diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index dc8eb2e..0cb227c 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -73,6 +73,41 @@ jobs: run: uv run python tools/check_suppression_owners.py - name: check file budgets run: uv run python tools/check_file_budgets.py + - name: The generated release contract is not stale + # W5: RELEASE_CONTRACT.md is generated from the ledger's required cells, + # the per-tool claim ledger and the live registry. Drift is a red gate, + # never a document that quietly stops matching the code. + run: uv run python tools/gen_release_contract.py --check + - name: Assert qualified runner + record identity + shell: bash + run: >- + python3 tools/runner_identity.py + --runner-os "${{ runner.os }}" --runner-arch "${{ runner.arch }}" + --runner-name "${{ runner.name }}" --expect-os "Linux" + --expect-arch "X64" --out "runner-identity.json" + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + python3 tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id quality --matrix-cell "default" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-quality + path: release-evidence + if-no-files-found: error # ── Unit + coverage floor on all three OSes, py3.11–3.13 ─────────────────── unit-tests: @@ -117,7 +152,34 @@ jobs: path: runner-identity.json - name: Run unit tests shell: bash - run: uv run pytest -m "not integration" -v --tb=short + run: >- + uv run pytest -m "not integration" -v --tb=short + --junitxml=junit.xml -o junit_family=xunit1 + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + uv run python tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id unit-tests --matrix-cell "${{ matrix.runner_os }}-${{ matrix.runner_arch }}-py${{ matrix.python-version }}" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --junit "junit.xml" + --artifact "junit=junit.xml" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-unit-${{ matrix.runner_os }}-${{ matrix.runner_arch }}-py${{ matrix.python-version }} + path: release-evidence + if-no-files-found: error coverage: name: coverage (${{ matrix.runner_os }}/${{ matrix.runner_arch }}) @@ -158,6 +220,7 @@ jobs: --cov-report=term-missing --cov-report=xml:coverage.xml --cov-fail-under=55 + --junitxml=junit.xml -o junit_family=xunit1 - name: Upload coverage report + runner identity uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: @@ -165,6 +228,32 @@ jobs: path: | coverage.xml runner-identity.json + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + uv run python tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id coverage --matrix-cell "${{ matrix.runner_os }}-${{ matrix.runner_arch }}" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --junit "junit.xml" + --artifact "coverage=coverage.xml" + --artifact "junit=junit.xml" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-coverage-${{ matrix.runner_os }}-${{ matrix.runner_arch }} + path: release-evidence + if-no-files-found: error # ── Real-Chrome evidence on all three OSes ───────────────────────────────── integration: @@ -227,6 +316,7 @@ jobs: uv run pytest -m "${{ runner.os == 'macOS' && 'integration and not transport' || 'integration' }}" -v --tb=short --timeout=180 + --junitxml=junit.xml -o junit_family=xunit1 env: DISPLAY: ${{ runner.os == 'Linux' && ':99' || '' }} STEALTH_MCP_BROWSER_SESSION_ROOT: ${{ runner.temp }}/stealth-mcp-session-root @@ -238,6 +328,34 @@ jobs: path: | chrome-identity.json runner-identity.json + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + uv run python tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id integration --matrix-cell "${{ matrix.runner_os }}-${{ matrix.runner_arch }}" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --chrome-identity "chrome-identity.json" + --chrome-launched + --junit "junit.xml" + --artifact "chrome-identity=chrome-identity.json" + --artifact "junit=junit.xml" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-integration-${{ matrix.runner_os }}-${{ matrix.runner_arch }} + path: release-evidence + if-no-files-found: error # The real-stdio journey. Linux/X64 and Windows/X64 only: macOS/ARM64 is a # KNOWN GAP (F-773 — Chrome under the detached backend completes no network @@ -289,7 +407,9 @@ jobs: # Session root at step level for the same runner-context reason as the # integration job above. shell: bash - run: uv run pytest -m transport -v --tb=short --timeout=300 + run: >- + uv run pytest -m transport -v --tb=short --timeout=300 + --junitxml=junit.xml -o junit_family=xunit1 env: DISPLAY: ${{ runner.os == 'Linux' && ':99' || '' }} STEALTH_MCP_BROWSER_SESSION_ROOT: ${{ runner.temp }}/stealth-mcp-session-root @@ -301,6 +421,34 @@ jobs: path: | chrome-identity.json runner-identity.json + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + uv run python tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id transport --matrix-cell "${{ matrix.runner_os }}-${{ matrix.runner_arch }}" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --chrome-identity "chrome-identity.json" + --chrome-launched + --junit "junit.xml" + --artifact "chrome-identity=chrome-identity.json" + --artifact "junit=junit.xml" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-transport-${{ matrix.runner_os }}-${{ matrix.runner_arch }} + path: release-evidence + if-no-files-found: error # ── The gate's own honesty surface ───────────────────────────────────────── # A required check whose ONLY job is to name what the gate does not verify, in @@ -313,6 +461,16 @@ jobs: name: known-gaps runs-on: ubuntu-latest steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.ref }} + - name: Assert qualified runner + record identity + shell: bash + run: >- + python3 tools/runner_identity.py + --runner-os "${{ runner.os }}" --runner-arch "${{ runner.arch }}" + --runner-name "${{ runner.name }}" --expect-os "Linux" + --expect-arch "X64" --out "runner-identity.json" - name: State the gaps run: | echo "::warning title=F-773::transport (macOS/ARM64) is NOT run. Chrome under the detached backend completes no network navigation on the hosted macOS runner. Cause unknown; 11 CI rounds, full elimination table in audit/stage2/finding_F773_macos_detached_navigation.md. Unknown whether real Macs are affected -- that needs a run on real hardware." @@ -324,6 +482,32 @@ jobs: echo " install-smoke : macOS/ARM64 -- wheel AND sdist (2 install+handshake cells)" echo "This gate makes NO claim about macOS navigation. Do not advertise one." echo "It DOES claim the built artifacts install and serve on macOS/ARM64." + echo "TOOL SURFACE: see RELEASE_CONTRACT.md -- the per-tool table is" + echo " generated from the ledger, and at this SHA the release-qualified" + echo " count is what the evidence supports, not the number of tools served." + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + python3 tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id known-gaps --matrix-cell "default" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-known-gaps + path: release-evidence + if-no-files-found: error # ── Offline stealth lane (W2 wires the edge; W4 lands the tests) ─────────── offline-stealth: @@ -370,9 +554,39 @@ jobs: # stopped collecting these tests would otherwise leave a green lane # asserting nothing, which is exactly the false green §8.1 forbids. shell: bash - run: uv run pytest -m "stealth and not online" -v --tb=short --timeout=300 + run: >- + uv run pytest -m "stealth and not online" -v --tb=short --timeout=300 + --junitxml=junit.xml -o junit_family=xunit1 env: DISPLAY: ${{ runner.os == 'Linux' && ':99' || '' }} + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + uv run python tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id offline-stealth --matrix-cell "${{ matrix.runner_os }}-${{ matrix.runner_arch }}" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --chrome-identity "chrome-identity.json" + --chrome-launched + --junit "junit.xml" + --artifact "chrome-identity=chrome-identity.json" + --artifact "junit=junit.xml" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-offline-stealth-${{ matrix.runner_os }}-${{ matrix.runner_arch }} + path: release-evidence + if-no-files-found: error # ── W3: build ONCE, verify and smoke THOSE EXACT FILES (gap G-C) ─────────── # The only `uv build` in the entire run. Everything downstream consumes the @@ -417,6 +631,37 @@ jobs: dist release-manifest.json if-no-files-found: error + - name: Assert qualified runner + record identity + shell: bash + run: >- + python3 tools/runner_identity.py + --runner-os "${{ runner.os }}" --runner-arch "${{ runner.arch }}" + --runner-name "${{ runner.name }}" --expect-os "Linux" + --expect-arch "X64" --out "runner-identity.json" + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + python3 tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id build-dist --matrix-cell "default" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --artifact "build-manifest=release-manifest.json" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-build-dist + path: release-evidence + if-no-files-found: error # Independent re-check of the DOWNLOADED bytes, plus the negative controls # that prove these rules can actually fail. Distinct from installation. @@ -499,6 +744,39 @@ jobs: path: | package-verify.json release-manifest.json + - name: Assert qualified runner + record identity + if: always() + shell: bash + run: >- + python3 tools/runner_identity.py + --runner-os "${{ runner.os }}" --runner-arch "${{ runner.arch }}" + --runner-name "${{ runner.name }}" --expect-os "Linux" + --expect-arch "X64" --out "runner-identity.json" + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + python3 tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id package-verify --matrix-cell "default" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --artifact "build-manifest=release-manifest.json" + --artifact "package-verify=package-verify.json" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-package-verify + path: release-evidence + if-no-files-found: error # Install the EXACT downloaded artifact into a fresh environment and run W1's # canonical journey through the launcher that install produced. Both @@ -591,6 +869,95 @@ jobs: chrome-identity.json runner-identity.json if-no-files-found: warn + - name: Emit this cell's release-evidence record + # W5: `always()` so a RED cell still writes a record saying it was + # red. The aggregate rejects any non-success terminal outcome, so + # this edge can only ever make the gate redder, never greener. + if: always() + shell: bash + run: >- + uv run python tools/release_evidence.py emit + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + --job-id install-smoke --matrix-cell "${{ matrix.kind }}-${{ matrix.cell.runner_os }}-${{ matrix.cell.runner_arch }}" + --terminal-outcome "${{ job.status }}" + --runner-identity "runner-identity.json" + --chrome-identity "chrome-identity.json" + ${{ matrix.cell.stages == 'full' && '--chrome-launched' || '' }} + --artifact "chrome-identity=chrome-identity.json" + --artifact "install-smoke=install-smoke-${{ matrix.kind }}.json" + --artifact "runner-identity=runner-identity.json" + - name: Upload this cell's release-evidence record + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-install-smoke-${{ matrix.kind }}-${{ matrix.cell.runner_os }}-${{ matrix.cell.runner_arch }} + path: release-evidence + if-no-files-found: error + + # ── W5: the fail-closed acceptance ledger (gap: "green" vs "claimable") ──── + # Every required cell above wrote one `release-evidence/v1` child record. This + # job re-reads ALL of them through the single parser and refuses to produce a + # successful aggregate on a missing/extra/duplicate child, a stale release SHA, + # a foreign workflow run, a non-success terminal outcome, a required node that + # skipped/xfailed/failed, a browser cell with no Chrome identity, a JUnit or + # artifact hash mismatch, a malformed/duplicate/absent MQ id, or a per-tool + # claim the run did not actually prove. + # + # It is a direct `needs:` edge of `release-gate` IN ADDITION to every child job + # it validates -- never instead of them. The ledger can only ever turn a green + # run red. + release-evidence: + name: release-evidence + if: always() + needs: + - quality + - unit-tests + - coverage + - integration + - transport + - known-gaps + - offline-stealth + - build-dist + - package-verify + - install-smoke + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.ref }} + - name: Install uv + uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 # v5.4.1 + - name: Set up Python 3.12 + run: uv python install 3.12 + - name: Install dependencies + # The aggregate derives the served tool surface from the live registry, + # so it needs the package importable -- the count is never typed. + run: uv sync --extra test --extra sentry + - name: Download every cell's evidence record + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: release-evidence-* + merge-multiple: true + path: release-evidence + - name: Aggregate the ledger (fail-closed) + shell: bash + run: >- + uv run python tools/release_evidence.py aggregate + --root release-evidence + --release-sha "$(git rev-parse HEAD)" + --run-id "${{ github.run_id }}" + --run-attempt "${{ github.run_attempt }}" + --event "${{ github.event_name }}" + - name: Upload the aggregate ledger + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: release-evidence-aggregate + path: release-evidence + if-no-files-found: error # ── Stable aggregate: the ONE public required check ──────────────────────── release-gate: @@ -607,6 +974,7 @@ jobs: - build-dist - package-verify - install-smoke + - release-evidence runs-on: ubuntu-latest steps: - name: Require every edge to have succeeded @@ -631,6 +999,7 @@ jobs: check build-dist "${{ needs.build-dist.result }}" check package-verify "${{ needs.package-verify.result }}" check install-smoke "${{ needs.install-smoke.result }}" + check release-evidence "${{ needs.release-evidence.result }}" if [ "$overall" -ne 0 ]; then echo "::error::release-gate: one or more required edges were not success" exit 1 diff --git a/.gitignore b/.gitignore index dc859e9..59e4915 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,13 @@ node_modules/ htmlcov/ .env +# plan_RELEASE W5: the release-evidence/v1 ledger is CI OUTPUT for one run +# (release-evidence//...). It is uploaded as a run artifact and +# re-read by the `release-evidence` job; a copy committed to the tree would be +# stale evidence for a SHA that is no longer HEAD — exactly what the aggregate's +# stale-SHA check exists to reject. +release-evidence/ + # Clone / large-response artifacts. The default output dir is now # ~/.stealth-mcp/element_clones (outside the repo), but never commit any that # land in-tree via an explicit relative path or a stale pre-fix run. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..165c802 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,132 @@ +# Changelog + +## 2.0.0 + +The first release since the foundational audit. It carries ~85 commits since 1.2.0 and +fixes four defects that were present in every prior release and invisible to the old +test suite, because none of them can be reproduced through the in-process test seam — +they only appear over the real stdio transport a client actually uses. + +### ⚠️ Breaking + +- **`STEALTH_MCP_SESSION_STORAGE_CAP_GB` is now `STEALTH_MCP_BROWSER_SESSION_STORAGE_CAP_GB`.** +- **`--session-cap-gb` is now `--browser-session-cap-gb`.** + + There is **no back-compat alias**. The old names are simply not read, so if you set + either one it stops taking effect **silently** on upgrade — your storage cap reverts to + the default. The rename removes a genuine ambiguity: "session" meant three different + things across this codebase (an MCP protocol session, a Claude Code session, and a + profile-backed browser session), and only the last one was ever meant here. + + Note the environment namespace is strict: an unrecognised `STEALTH_MCP_*` variable is + rejected at startup rather than ignored, so a stale name fails loudly at the *next* + restart even though the setting itself silently lapsed. + +### Fixed + +- **Browsers were destroyed every ~2 seconds over real stdio.** FastMCP runs the server + lifespan once per *MCP session*, and the liveness watchdog's probe sessions each re-ran + orphan recovery and its destructive teardown — killing every live browser instance + belonging to the real session. Anyone driving this server the normal way (stdio proxy → + detached backend) had instances disappear underneath them. The lifespan is now + session-reentrant. +- **`list_tabs` raised a bare `TypeError` after any `close_tab`.** nodriver re-adds + rediscovered targets as raw `Connection` objects, which are not awaitable, and the tool + awaited each one. Once a tab had been closed the failure was permanent for that + browser, not transient. +- **Every navigation after a `close_tab` silently switched tabs and leaked one.** The + same root cause, but swallowed by a broad exception handler: the tracked tab was found + correctly, the liveness check on it raised, and the handler concluded the tab was + "missing or invalid" and replaced it — without closing the original. No error surfaced; + navigation simply happened in a different tab each time, and the abandoned tabs + accumulated. +- **`close_tab` returned `False` for a closeable tab**, and **`switch_to_tab` failed to + activate**, for the same class of rediscovered target. Both now address the target by + id through CDP, which works regardless of object type. +- **Headless mode advertised `HeadlessChrome` in its User-Agent.** That is the cheapest + bot check that exists — one server-side substring test, before any JavaScript runs — + and it contradicted the product's central claim. A default headless spawn now presents + the same User-Agent the same binary presents headed, on the page, on the wire, and at + the CDP level. An explicitly supplied `user_agent` still wins. See *Known limitations* + for what this does **not** fix. +- **Every spawn enabled catch-all network interception, even with no hooks defined.** + Chrome paused every request and waited for a resume that only the hook handler would + send, so all traffic paid a pause plus a CDP round-trip for no benefit. Interception is + now armed only when there is something to intercept — and, relatedly, a hook created + *after* spawn now arms interception through the same path instead of relying on the + catch-all's accidental coverage. +- **Selector resolution could hit stale-node `-32000` errors under DOM churn**, because + nodriver's `select`/`find`/`query_selector` are not atomic. All selector resolution now + routes through a single resolver that survives document-node invalidation. +- A Tier-A pass on silent-correctness and "lying success" defects — cases where a tool + reported success without having done the thing (PR #41). + +### Added + +- A **three-OS release gate** (Ubuntu x64, Windows x64, macOS ARM64) that exercises the + real stdio transport against real Chrome, asserts the exact Chrome binary identity, and + gates on a single aggregate check. Previous releases were verified on Ubuntu only. +- **Build-once packaging.** The distribution is built exactly once per commit, hashed, + verified, and installed from that same artifact in smoke tests; the publish step + downloads those bytes, re-checks their SHA-256, and uploads them without rebuilding. + What was tested is what ships. +- A **deterministic offline stealth suite** asserting anti-detection invariants against a + vanilla-Chrome control, so a regression that reintroduces an automation tell fails the + build. +- The **source distribution shrank from 15 MB to 592 KB.** It had been shipping 12.8 MB + of demo media and 2.8 MB of internal audit documents. The wheel — what `pip` and `uvx` + actually install — is unchanged at 192 KB; this only affects installing from source. + +### Known limitations + +Stated explicitly rather than by omission. + +- **macOS: navigation is unverified.** On GitHub-hosted macOS/ARM64 runners, Chrome + launched by the detached backend completes no network navigation (reproducible 11/11); + a connection to a *closed* port hangs rather than being refused, so the request never + reaches the network stack. The cause is unknown and it has **never been reproduced on a + real Mac** — hosted runners differ in ways that plausibly matter. The gate therefore + excludes the macOS transport cell and runs macOS install-smoke without navigation. This + release makes **no claim that macOS navigation works, and none that it is broken.** + Linux x64 and Windows x64 are verified. +- **Headless is not "undetectable".** The User-Agent fix closes the cheapest and most + widely deployed check, but supplying a User-Agent override makes Chrome blank its + high-entropy client hints (`architecture`, `bitness`, `platformVersion`, + `uaFullVersion`, `fullVersionList`). Low-entropy hints and every `sec-ch-ua*` header on + the wire remain correct and coherent, so the residue is reachable only from JavaScript + that explicitly calls `getHighEntropyValues()` — a strictly smaller and more expensive + tell than the one it replaces, but a real one. +- **`switch_to_tab` can still store a rediscovered target** as an instance's main tab. + Activation is fixed; the storage path is not. It fails loudly if it fires. +- **The HTTP transport is unauthenticated and loopback-default by design.** All + verification here covers stdio; stdio evidence licenses no HTTP claim. +- **Not evidenced in this release:** scheduled drift observation, deterministic + site-breadth corpus, manual-QA parity tripwire, performance and resource budgets, + fault-injection/resilience, runnable-documentation checks, the security/trust-boundary + matrix, wire concurrency/cancellation and independent-client interoperability, + upgrade/migration smoke, failure-observability, and worker/PWA/internationalized site + shapes. These are planned work that has **not** been performed — do not read their + absence as a passing result. +- **Per-tool verification depth varies, and the release contract says so per tool.** All + 94 tools have end-to-end coverage driving real Chrome against a local fixture, + enforced by a set-equality tripwire (94 covered, 0 exempt). But only `set_cookie`, + `get_cookies` and `clear_cookies` are additionally verified over the **real stdio + transport** your client actually speaks; the rest are exercised through an in-process + test seam that bypasses the wire. That distinction is not academic — every one of the + transport bugs fixed above was invisible to the seam. It is a gap in test placement + rather than a known defect in those 91 tools, which is why the contract calls them + *served* rather than *release-qualified*. +- A known flake exists in one packaging smoke cell (`install-smoke (sdist Linux/X64)`) on + Chrome cold-spawn; it passes on re-run. + +### Upgrading from 1.x + +1. Rename `STEALTH_MCP_SESSION_STORAGE_CAP_GB` → `STEALTH_MCP_BROWSER_SESSION_STORAGE_CAP_GB` + and `--session-cap-gb` → `--browser-session-cap-gb` wherever you set them. +2. Pin the new version, e.g. `uvx stealth-chrome-devtools-mcp==2.0.0`. +3. Restart the backend. Code changes apply via a fresh backend process — the singleton is + version-gated, so an old backend is evicted rather than reused. + +## 1.2.0 and earlier + +Not tracked in this file; see the repository history. diff --git a/README.md b/README.md index 512597e..e758e98 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Add to your MCP config (`claude_desktop_config.json`, `.claude/settings.json`, e "mcpServers": { "stealth-chrome-devtools-mcp": { "command": "uvx", - "args": ["stealth-chrome-devtools-mcp==1.2.0"] + "args": ["stealth-chrome-devtools-mcp==2.0.0"] } } } @@ -58,7 +58,7 @@ Add to your MCP config (`claude_desktop_config.json`, `.claude/settings.json`, e Or install via pip: ```bash -pip install stealth-chrome-devtools-mcp==1.2.0 +pip install stealth-chrome-devtools-mcp==2.0.0 ``` ### Local Development diff --git a/RELEASE_CONTRACT.md b/RELEASE_CONTRACT.md new file mode 100644 index 0000000..0ded71f --- /dev/null +++ b/RELEASE_CONTRACT.md @@ -0,0 +1,391 @@ + + + +# Release contract — version 2.0.0 + +This is the contract for the version recorded in `pyproject.toml` at +this commit. A tagged run fails unless the tag equals that version, +so the two cannot disagree. + +What a green `release-gate` check does and does not authorize. Every +number below is derived: the served-tool count from the live +`SECTION_TOOLS` registry, the qualified count from the per-tool claim +ledger, and the matrix from the required cells of the +`release-evidence/v1` aggregate. Nothing here is hand-typed, and no +prose document — plan, finding, or README — can qualify anything. + +> **At the release SHA recorded in the ledger, this gate qualifies 3 of the 94 served MCP tools**, on the cells each +> row names. + +**Read that number carefully, in both directions.** It does *not* say +91 tools are untested. Every served tool +is driven against **real Chrome** by the E2E suite — a set-equality +tripwire keeps that coverage complete, and §5 shows each tool's actual +evidence. What those tools lack is a *per-tool* assertion over the +**stdio transport a user actually speaks**, because the suite drives +them through the in-process function seam instead. plan_RELEASE §2.5 +does not let in-process evidence license a transport claim, so they +are `served-unqualified`: tested, but not proved at the wire. Closing +that is a question of where tests run, not of a product defect — see +F-776. + +### Breaking change from 1.x — read this before upgrading + +Two knobs were **renamed with no back-compatible alias**: + +| 1.x | now | effect if you keep using the old one | +|---|---|---| +| `STEALTH_MCP_SESSION_STORAGE_CAP_GB` | `STEALTH_MCP_BROWSER_SESSION_STORAGE_CAP_GB` | the variable is ignored — the cap silently returns to its default | +| `--session-cap-gb` | `--browser-session-cap-gb` | the CLI rejects the unknown flag | + +The environment variable is the dangerous one: nothing errors, your +configured storage cap simply stops applying. Rename it before you +upgrade. + +### What this contract is NOT + +plan_RELEASE reserved a specific property — *a green gate is a +faithful stand-in for a manual pass, so you may push blind* — for a +workstream that has **not run**. That property rests on three things: +manual-QA parity, proven flake-freedom, and mutation-informed test +strength. **None of the three is established here**, and one required +cell has a known flake (below). This gate is strong on what it covers: +the real stdio wire path, three OSes, the exact published artifacts, +and offline stealth invariants. It is not a substitute for a human +release pass, and this document does not authorize a blind push. + +## 1. The qualified matrix + +Exactly these GitHub-hosted cells, and nothing else. Each one writes a +`release-evidence/v1` child record; the `release-evidence` job refuses to +aggregate unless every cell below is present, current, and successful. + +| Job | Matrix cell | Runner | Python | Chrome | What the cell proves | +|---|---|---|---|---|---| +| `quality` | `default` | Linux/X64 | image `python3` (recorded per run) | — | lint/type/vulture/owner/budget gates | +| `known-gaps` | `default` | Linux/X64 | image `python3` (recorded per run) | — | the declared gaps, in the check list | +| `build-dist` | `default` | Linux/X64 | image `python3` (recorded per run) | — | the ONE build + its hashed manifest | +| `package-verify` | `default` | Linux/X64 | image `python3` (recorded per run) | — | downloaded-bytes re-check + three bite proofs | +| `unit-tests` | `Linux-X64-py3.11` | Linux/X64 | 3.11 | — | hermetic unit suite (`-m 'not integration'`) | +| `unit-tests` | `Windows-X64-py3.11` | Windows/X64 | 3.11 | — | hermetic unit suite (`-m 'not integration'`) | +| `unit-tests` | `macOS-ARM64-py3.11` | macOS/ARM64 | 3.11 | — | hermetic unit suite (`-m 'not integration'`) | +| `unit-tests` | `Linux-X64-py3.12` | Linux/X64 | 3.12 | — | hermetic unit suite (`-m 'not integration'`) | +| `unit-tests` | `Windows-X64-py3.12` | Windows/X64 | 3.12 | — | hermetic unit suite (`-m 'not integration'`) | +| `unit-tests` | `macOS-ARM64-py3.12` | macOS/ARM64 | 3.12 | — | hermetic unit suite (`-m 'not integration'`) | +| `unit-tests` | `Linux-X64-py3.13` | Linux/X64 | 3.13 | — | hermetic unit suite (`-m 'not integration'`) | +| `unit-tests` | `Windows-X64-py3.13` | Windows/X64 | 3.13 | — | hermetic unit suite (`-m 'not integration'`) | +| `unit-tests` | `macOS-ARM64-py3.13` | macOS/ARM64 | 3.13 | — | hermetic unit suite (`-m 'not integration'`) | +| `coverage` | `Linux-X64` | Linux/X64 | 3.12 | — | per-OS coverage floor (no merged report hides a red OS) | +| `coverage` | `Windows-X64` | Windows/X64 | 3.12 | — | per-OS coverage floor (no merged report hides a red OS) | +| `coverage` | `macOS-ARM64` | macOS/ARM64 | 3.12 | — | per-OS coverage floor (no merged report hides a red OS) | +| `integration` | `Linux-X64` | Linux/X64 | 3.12 | launched + identity asserted | real-Chrome integration suite + Chrome identity | +| `integration` | `Windows-X64` | Windows/X64 | 3.12 | launched + identity asserted | real-Chrome integration suite + Chrome identity | +| `integration` | `macOS-ARM64` | macOS/ARM64 | 3.12 | launched + identity asserted | real-Chrome integration suite + Chrome identity MINUS the transport journey (F-773) | +| `offline-stealth` | `Linux-X64` | Linux/X64 | 3.12 | launched + identity asserted | offline stealth predicates and their failing controls | +| `offline-stealth` | `Windows-X64` | Windows/X64 | 3.12 | launched + identity asserted | offline stealth predicates and their failing controls | +| `offline-stealth` | `macOS-ARM64` | macOS/ARM64 | 3.12 | launched + identity asserted | offline stealth predicates and their failing controls | +| `transport` | `Linux-X64` | Linux/X64 | 3.12 | launched + identity asserted | real-stdio JSON-RPC journey against real Chrome | +| `transport` | `Windows-X64` | Windows/X64 | 3.12 | launched + identity asserted | real-stdio JSON-RPC journey against real Chrome | +| `install-smoke` | `wheel-Linux-X64` | Linux/X64 | 3.12 | launched + identity asserted | clean install of the exact wheel + journey | +| `install-smoke` | `wheel-Windows-X64` | Windows/X64 | 3.12 | launched + identity asserted | clean install of the exact wheel + journey | +| `install-smoke` | `wheel-macOS-ARM64` | macOS/ARM64 | 3.12 | resolved only (no launch) | clean install of the exact wheel + handshake only (NO navigation, F-773) | +| `install-smoke` | `sdist-Linux-X64` | Linux/X64 | 3.12 | launched + identity asserted | clean install of the exact sdist + journey | +| `install-smoke` | `sdist-Windows-X64` | Windows/X64 | 3.12 | launched + identity asserted | clean install of the exact sdist + journey | +| `install-smoke` | `sdist-macOS-ARM64` | macOS/ARM64 | 3.12 | resolved only (no launch) | clean install of the exact sdist + handshake only (NO navigation, F-773) | + +The exact Chrome Stable executable path and version are **not** written +into this document: they are image-provided and change under us. They +are recorded per run in +`release-evidence//integration/.json`, and the +integration cell asserts that production's own auto-discovery and the +CDP `Browser.getVersion` of the launched browser agree with them. + +## 2. What the matrix explicitly excludes + +These are not 'probably fine'. They are **untested**: + +- untested Linux distributions (the gate runs the hosted Ubuntu image + only) and any self-hosted runner; +- Windows ARM64 and Intel macOS; +- IPv6-only loopback; +- non-Stable Chrome channels (Beta, Dev, Canary, Chromium, Edge); +- future GitHub runner images — a qualification is a statement about + the image identity recorded in that run's ledger, not a standing + promise about `*-latest`. + +The phrase 'Linux, Windows and macOS' may only ever appear here with +those qualifiers attached: the qualified cells are **Ubuntu x64, +Windows x64 and macOS ARM64**, each as provided by the GitHub-hosted +image whose id the ledger records. + +## 3. Transport qualification + +### stdio (the transport a user actually gets) + +Qualified on: **Linux/X64**, **Windows/X64**. + +The real-stdio lane spawns the installed console launcher, completes +the `initialize` handshake, lists tools, and drives one canonical +journey against real Chrome. macOS/ARM64 is **excluded** from this +lane under F-773 — excluded rather than xfailed, because an xfail +would let a green gate imply macOS coverage. + +That journey is a *representative* proof that the wire path works. It +is deliberately **not** per-tool evidence (plan_RELEASE §2.5), which is +why it qualifies the transport but no individual tool. + +### HTTP (described, not qualified) + +The server also supports `--transport http`. It is **unauthenticated** +by design and binds loopback by default. It is described here, not +qualified: no HTTP claim is derived from stdio evidence, and the gate +runs no live HTTP acceptance test. Anything able to reach that port +drives the browser with the caller's full privileges. + +## 4. Upgrade qualification + +**None.** W14 has not run: this contract makes no upgrade, migration, +or rollback claim from the literal N-1 stable tag or from any other +version. Installing over an existing installation is unqualified. + +## 5. The served tool surface + +- served by the registry: **94** +- `release-qualified-success`: **3** +- `served-unqualified`: **91** +- `not-served`: **0** + +A tool is `release-qualified-success` only when a row names the precise +user outcome, a fully-qualified passing node, the required transport, the +fixture or site shape, and the required OS cells — and the ledger shows +each of them as current-run success evidence. A schema or type assertion, +a `.fn`-only call, the representative journey, an error-only test, an +exemption, or a characterization **cannot** satisfy that bar. + +A qualified row is qualified **only on the cells its claim names**, and +the ledger enforces that: a `stdio` claim must be evidenced by the +transport lane, which runs on **Linux/X64 and Windows/X64 only**. macOS +ARM64 is excluded from that lane under F-773, so no per-tool stdio claim +in this document is qualified on three cells — every one of them is +qualified on exactly two. + +`served-unqualified` does not mean broken, and it does not mean untested. +It means: the server serves the tool, the suite exercises it against real +Chrome, and the gate at this SHA does not prove the user-visible outcome +*over the transport the user speaks*. The evidence column below states +which of those is true for each row — it is a description, never a claim. +A row reading **no execution evidence** would be materially weaker than +the rest; if none appears below, no served tool is in that state. + +Unless a row names a specific defect, its tracking id is **F-776** and its +impact is the shared one: no per-tool real-transport success assertion +exists at this SHA. + +| Tool | Section | State | Strongest current evidence | Tracking id | +|---|---|---|---|---| +| `close_instance` | browser-management | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-775d | +| `get_instance_state` | browser-management | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `go_back` | browser-management | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `go_forward` | browser-management | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `list_instances` | browser-management | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `navigate` | browser-management | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `reload_page` | browser-management | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `spawn_browser` | browser-management | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `call_javascript_function` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | TRUST-BOUNDARY | +| `create_persistent_function` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `create_python_binding` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | TRUST-BOUNDARY | +| `discover_global_functions` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `discover_object_methods` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `execute_cdp_command` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | TRUST-BOUNDARY | +| `execute_function_sequence` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `execute_python_in_browser` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | TRUST-BOUNDARY | +| `get_execution_contexts` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_function_executor_info` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `inject_and_execute_script` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | TRUST-BOUNDARY | +| `inspect_function_signature` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `list_cdp_commands` | cdp-functions | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `clear_cookies` | cookies-storage | release-qualified-success | stdio — `tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip` | — | +| `get_cookies` | cookies-storage | release-qualified-success | stdio — `tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip` | — | +| `set_cookie` | cookies-storage | release-qualified-success | stdio — `tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip` | — | +| `clear_debug_view` | debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `export_debug_logs` | debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_debug_lock_status` | debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_debug_view` | debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `validate_browser_environment_tool` | debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `create_dynamic_hook` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `create_simple_dynamic_hook` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_dynamic_hook_details` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_hook_common_patterns` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_hook_documentation` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_hook_examples` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_hook_requirements_documentation` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `list_dynamic_hooks` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `remove_dynamic_hook` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `validate_hook_function` | dynamic-hooks | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `clone_element_complete` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_complete_element_cdp` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_animations` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_assets` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_events` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_structure` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_styles` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_styles_cdp` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_related_files` | element-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `click_element` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | E8-2 | +| `execute_script` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | TRUST-BOUNDARY | +| `get_element_state` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | E7-6 | +| `get_page_content` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `paste_text` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `query_elements` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `scroll_page` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `select_option` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | E8-1 | +| `take_screenshot` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `type_text` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | E7-1/E8-3/E8-4 | +| `upload_file` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `wait_for_element` | element-interaction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `cleanup_clone_files` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `clone_element_to_file` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_complete_element_to_file` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_animations_to_file` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_assets_to_file` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_events_to_file` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_structure_to_file` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `extract_element_styles_to_file` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `list_clone_files` | file-extraction | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `export_network_data` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_network_capture_filters` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_request_details` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_response_content` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `get_response_details` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `import_network_data` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `list_network_requests` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `modify_headers` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-165 | +| `search_network_requests` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `set_network_capture_filters` | network-debugging | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `clear_all_elements` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `clear_stored_element` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `clone_element_progressive` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `expand_animations` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `expand_children` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `expand_css_rules` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `expand_events` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `expand_pseudo_elements` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `expand_styles` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `list_stored_elements` | progressive-cloning | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `close_tab` | tabs | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-775b/macOS-close-flake | +| `get_active_tab` | tabs | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `list_tabs` | tabs | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `new_tab` | tabs | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-776 | +| `switch_tab` | tabs | served-unqualified | E2E-covered against **real Chrome** through the in-process seam; not verified over the stdio transport | F-775c-residual | + +### Tools whose row carries a specific defect + +| Tool | Tracking id | User impact | +|---|---|---| +| `close_instance` | F-775d | Teardown uses the verified CDP call but has no dedicated pin - it is exercised only indirectly by integration teardown, so a regression here would surface as someone else's flake. | +| `call_javascript_function` | TRUST-BOUNDARY | Invokes arbitrary page functions by design. Trust boundary unverified (W12 not run). | +| `create_python_binding` | TRUST-BOUNDARY | Exposes a Python callable to page JavaScript by design. Trust boundary unverified (W12 not run). | +| `execute_cdp_command` | TRUST-BOUNDARY | Sends arbitrary CDP commands by design - the widest surface in the server. Trust boundary unverified (W12 not run). | +| `execute_python_in_browser` | TRUST-BOUNDARY | Evaluates caller-supplied Python in the server process by design. Trust boundary unverified (W12 not run). | +| `inject_and_execute_script` | TRUST-BOUNDARY | Injects and runs caller-supplied JavaScript by design. Trust boundary unverified (W12 not run). | +| `click_element` | E8-2 | Clicking a disabled control returns True. The caller cannot distinguish 'clicked' from 'ignored by the browser'. | +| `execute_script` | TRUST-BOUNDARY | Executes caller-supplied JavaScript in the page by design. The trust boundary is unverified: W12 has not run. | +| `get_element_state` | E7-6 | Reports HTML attributes, not live DOM properties, so a value changed by script is reported stale. | +| `select_option` | E8-1 | A select whose page script re-declares a const swallows the change: the tool returns True while the option did not change. Silent wrong-success. | +| `type_text` | E7-1/E8-3/E8-4 | clear_first bypasses readonly (E8-3); contenteditable is not cleared before typing (E7-1); range/color/date inputs are not reachable through this path at all (E8-4). | +| `modify_headers` | F-165 | Duplicate header names are mishandled in the rewrite loop. | +| `close_tab` | F-775b/macOS-close-flake | Closing by target id is fixed (FIX-F). One macOS CI attempt returned True while the target survived a 10s poll; that observation is NOT reproducible and is NOT recorded as closed. | +| `switch_tab` | F-775c-residual | Activation is fixed (FIX-F), but the instance's main tab is still stored from the raw browser.tabs entry, which can be a Connection rather than a Tab. Loud if it fires, and it seeds the F-775a family. | + +## 6. Limitations register + +Everything a reader must know before trusting a green check. Rows are not +removed to shorten the document; a row leaves only when the thing it +describes actually closes. + +| Id | Area | Status | User impact | Evidence status | +|---|---|---|---|---| +| E8-1 | interaction / `select_option` | open | A page script that re-declares a const makes the change a silent no-op while the tool returns True. | characterization pin only — never a success claim. | +| E8-2 | interaction / `click_element` | open | Clicking a disabled control returns True; the caller cannot tell 'clicked' from 'ignored'. | characterization pin only. | +| E8-3 | interaction / `type_text` | open | `clear_first` bypasses a readonly field instead of refusing it. | characterization pin only. | +| E8-4 | interaction / typed inputs | open | range, color and date inputs are not reachable through the typing path. | characterization pin only. | +| E7-1 | interaction / contenteditable | open | `type_text` does not clear a contenteditable before typing. | characterization pin only. | +| E7-6 | state / `get_element_state` | open | Reports HTML attributes rather than live DOM properties, so a script-updated value reads stale. | characterization pin only. | +| F-165 | network / `modify_headers` | open | Duplicate header names are mishandled by the rewrite loop. | characterization pin only. | +| F-181 | internal / stale live node | open (internal) | An internal `-32000` stale-node path. It is NOT a public acceptance surface and is unsupported; no user-facing behaviour is claimed for it. | characterization pin only. | +| close-path flake | lifecycle / close | open | The close path has a known flake history; teardown ordering is not deterministic under all timings. | characterization pin only. | +| macOS `close_tab` observation | lifecycle / `close_tab` on macOS | unreproduced | One CI attempt returned True while the target survived a 10s poll. Not reproducible (Windows local and all three CI cells clean on re-run). | NOT characterized and NOT recorded as closed — an unexplained single observation, listed so it is not forgotten. | +| F-773 | platform / macOS ARM64 navigation | open, cause unknown | Under the detached backend on hosted macOS runners, Chrome completes no network navigation (a connection to a CLOSED port hangs ~35s while about:blank returns in ~1.1s). Reproduced 11/11 on CI; never reproduced on real Mac hardware. | The macOS transport cell is EXCLUDED (not xfailed) and both macOS install-smoke cells are NO-NAVIGATION partial. This contract therefore claims neither that macOS navigation works NOR that it is broken. | +| F-774 | stealth / UA client hints | open | The headless User-Agent override blanks the high-entropy UA client hints (`architecture`, `bitness`, `platformVersion`, `uaFullVersion`, `fullVersionList`). `brands`/`mobile`/`platform` and every `sec-ch-ua*` wire header stay correct. | Measured and recorded. Headless is therefore NOT claimed undetectable; the residue is strictly smaller than the leak it replaced (F-770). | +| F-775c residual | tabs / `switch_tab` | open | Activation is fixed, but the instance's main tab is still stored from the raw `browser.tabs` entry, which can be a `Connection`. Loud if it fires; it seeds the F-775a class. | routed, not fixed. | +| F-775d | lifecycle / `close_instance` | declared gap | Teardown uses the same verified CDP call as the fixed siblings but has no dedicated pin; it is exercised only indirectly by integration teardown. | no independent evidence — declared, not claimed. | +| `_replace_main_tab` residual | tabs / instance main-tab identity | open | `browser_manager.py` awaits `browser.get(..., new_tab=True)`, which returns whatever `browser.targets` holds for that id — a `Connection` if `update_targets()` won the race. Same family as F-775, lower severity, outside FIX-F's four sites. | routed, not fixed. | +| F-776 | evidence / per-tool transport coverage | open (opened by W5) | Only the cookie round-trip tools have a per-tool real-transport success assertion. Every other served tool IS exercised against real Chrome — through the in-process seam, plus the representative journey and the in-memory client — but none of those may license a per-tool transport claim under §2.5. The gap is where the tests run, not whether they run. | See audit/stage2/finding_F776_no_per_tool_transport_evidence.md. Closing it means moving/adding per-tool assertions into the transport lane, not relabelling what exists. | +| F-777 | test harness / `get_cookies` through the `.fn` seam | open (test infrastructure, not a user-facing defect) | Called through the in-process `.fn` seam the E2E suite uses, both CDP retrieval paths hang (~30s, no return) AND the tab's CDP connection is poisoned: the NEXT call on that tab dies with a 10s timeout. Measured on the same tool and the same Chrome that succeed over real stdio, so the blast radius is the seam, not the product. | The user-facing path is evidenced (the qualified cookie row). No E2E test may call `get_cookies` through the `.fn` seam; the tool's coverage lives in the transport lane. See audit/stage2/finding_F777_get_cookies_fn_seam_hang.md. | +| F-778 | types / `get_cookies` return shape | open (cosmetic) | `get_cookies` is declared `-> list[dict[str, Any]]` but returns nodriver `cdp.network.Cookie` dataclasses. The WIRE shape is correct — pydantic serializes them into proper JSON objects in `structuredContent` — so a client sees real cookie objects; only fastmcp's `.data` reconstruction is opaque (`[Root()]`). | No user impact measured. See audit/stage2/finding_F778_get_cookies_return_type_mismatch.md. | +| F-779 | gate reliability / macOS integration teardown | open, MEASURED at ~1 run in 4 (2 failures in 8 consecutive runs) | `integration (macOS/ARM64)` intermittently fails with `Event loop is closed` at teardown, reddening `release-evidence` and the `release-gate` aggregate with it. Code independence is established twice by different methods: (1) a re-run against a BYTE-IDENTICAL tree (an empty commit built from the same tree object) returned 32/32 success; (2) a later failure landed on a commit whose ENTIRE content is one markdown file. Across six consecutive runs on the W5 line the cell failed twice, with successes bracketing each failure. The mechanism is undiagnosed: job logs require admin rights, so only check-run annotations were available, and they carry the message but no traceback. | This is the decisive limitation on the release's headline goal. The aggregate requires EVERY edge green, so a ~25% failure on one cell puts the headline check red roughly one run in three regardless of the other 31 jobs. plan_RELEASE §0.2 makes flake-freedom one of the three properties behind 'green ⇒ blindly pushable' — so while this is open, a green check may be read as evidence about THIS run and nothing more, and a red one may not be read as evidence about your change at all. That is the exact ambiguity a release gate exists to remove. W8 owns flake quarantine and has not run. See audit/stage2/finding_F779_macos_integration_teardown_flake.md. | +| Linux cold-spawn flake | gate reliability | open, observed repeatedly | Chrome intermittently refuses the first connect on the Linux runner (`Failed to connect to browser`) inside the canonical journey, and the harness's bounded warmup retry does not always absorb it. Observed in `install-smoke (sdist Linux/X64)`, and later in the SAME run in both `transport (Linux/X64)` and `install-smoke (wheel Linux/X64)` — while `transport (Windows/X64)` and every other cell passed, and the cookie test in the very same Linux transport job spawned Chrome successfully seconds later. It is a cold-start race, not a code defect, and it lands on a cell that carries a qualified claim. | This gate is therefore NOT proven flake-free, and the flake can hit a cell whose evidence a claim depends on. plan_RELEASE §0.2 makes flake-freedom one of the three properties behind 'green ⇒ blindly pushable'; W8 owns flake quarantine and has not run, so no flake-freedom claim is made here. | +| missing interaction surface | tools / interaction census | excluded | There are no double-click, right-click, drag, or native-dialog tools. A workflow needing them cannot be automated by this server. | documented absence — plan_RELEASE §1.2 forbids building them here. | +| HTTP transport | trust boundary / transport | excluded from qualification | `--transport http` is UNAUTHENTICATED by design and binds loopback by default. Anything that can reach the port drives the browser. | stdio evidence never licenses an HTTP claim; the gate qualifies stdio only. | +| code-execution surface | trust boundary / exec | excluded from qualification | `execute_script`, `inject_and_execute_script`, `call_javascript_function`, `execute_cdp_command`, `execute_python_in_browser` and `create_python_binding` run caller-supplied code by design. | W12 (security/trust boundary) has NOT run; no security property is claimed. | +| architecture / channel | matrix | excluded | Untested Linux distributions, self-hosted runners, Windows ARM64, Intel macOS, IPv6-only loopback, non-Stable Chrome channels and future runner images are all outside the qualified matrix. | no evidence exists for any of them; a runner without a GitHub-hosted image identity is rejected by the ledger. | +| native IME | input / internationalization | excluded | Native IME/composition UI is not driven; only synthetic input is. | W16 has NOT run. | +| live public web | site shapes | informational only | All deterministic evidence uses the local fixture app. No public site, detector score, or arbitrary-site behaviour is qualified. | the live tier is read-only observation and is not part of the gate. | +| W6 scheduled observation | NOT EVIDENCED in this release | no evidence exists | There is no scheduled canary, so nothing observes drift between releases. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W7 site breadth | NOT EVIDENCED in this release | no evidence exists | Deterministic dynamic-site breadth is unqualified. Specifically unsupported or unqualified: stale live handles; recursive or frame-targeted content and interaction; redirect chains and loops; typed loading-failure and truncation; downloads; MCP-network SSE/WS detail; and generic or closed shadow-root access. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W8 manual-QA parity | NOT EVIDENCED in this release | no evidence exists | MQ steps are not yet mapped to runtime evidence, so the ledger's required-MQ set is empty and the manual protocol still governs. | Nothing in this release verifies manual-QA parity, flake-freedom, or test strength; the reader may not infer that any of them was checked. The ledger enforces MQ ids structurally, but no mapping exists to enforce yet. | +| W9 performance | NOT EVIDENCED in this release | no evidence exists | No latency, memory, or large-payload budget is asserted. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W10 resilience | NOT EVIDENCED in this release | no evidence exists | Crash, hang, tab-loss and network-fault recovery are unqualified. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W11 executable docs | NOT EVIDENCED in this release | no evidence exists | Documentation examples are not executed, so a doc example may be stale. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W12 security | NOT EVIDENCED in this release | no evidence exists | No filesystem, upload/export, redaction, or bind-address property is verified. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W13 wire semantics | NOT EVIDENCED in this release | no evidence exists | Concurrency, correlation, cancellation, disconnect and framing are unqualified, and no independent MCP client has driven this server in the gate. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W14 upgrade / rollback | NOT EVIDENCED in this release | no evidence exists | NO upgrade, migration, or rollback claim of any kind is made — not from the literal N-1 stable tag, not from any other version. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W15 observability | NOT EVIDENCED in this release | no evidence exists | Failure diagnostics are not verified as actionable, bounded, or redacted. | Nothing in this release verifies it; the reader may not infer that it was checked. | +| W16 state / PWA / i18n | NOT EVIDENCED in this release | no evidence exists | Workers, persistent state, PWA behaviour and Unicode/RTL round-trips are unqualified. | Nothing in this release verifies it; the reader may not infer that it was checked. | + +## 7. The ceiling + +This contract does **not** promise that the server works on any site, +or that it will keep working. The open web and Chrome change +adversarially; a green gate is a statement about one SHA, one set of +recorded runner images, one recorded Chrome Stable build, and the +fixture shapes named above. + +It does **not** promise universal undetectability. The offline stealth +predicates passed their failing controls on all three cells — that is +sensitivity, not invisibility — and F-774 records a real residual +client-hint tell in the headless UA override. + +It does **not** claim the gate is flake-free — and on current +measurement it is not close. TWO distinct gate flakes are on +record. A Chrome cold-spawn race on the Linux runner has failed +first attempts in three different cells, including +`transport (Linux/X64)` — one of the two cells that carry the +qualified stdio claims. And F-779: `integration (macOS/ARM64)` +fails at teardown with `Event loop is closed` in **2 of 8** +consecutive runs, taking the `release-gate` aggregate down with it +each time. Its code independence is settled — one failure landed on +a commit containing nothing but a markdown file. + +Read that consequence carefully, because it is the honest headline +of this release: **the aggregate check is currently red about one +run in three for reasons that have nothing to do with the change +under review.** plan_RELEASE §0.2 makes flake-freedom one of the +three properties behind 'green ⇒ blindly pushable'. Until F-779 is +closed, a green check is evidence about that run only, and a red +check is not by itself evidence about your change. The workstream +that owns flake quarantine has not run. + +A green check is also evidence about ONE run attempt. The evidence +ledger binds every cell record to a single `run_id` + `run_attempt`, +so re-running only the failed jobs cannot produce a certificate — +the surviving records belong to the earlier attempt and are refused. +Re-run the whole workflow, or the gate stays red on identity, not on +the retried failure. + +Live public sites and detector scores are read-only informational +observations. They never license a deterministic claim, and no such +observation runs in this gate. + +If a claim you need is not written above, the honest answer is that +this gate does not make it. diff --git a/RUNBOOK.md b/RUNBOOK.md index 0a8024e..31dfae8 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -42,7 +42,7 @@ and teardown live in the backend and are reused from the eviction path. backend : running (responsive) on port 19222 pid : 12345 log : C:\Users\you\.stealth-mcp\logs\backend-12345.log -version : 1.2.0 +version : 2.0.0 browser-session root: C:\stealth-mcp-browser-sessions (exists: True) clone cap : 10.0 GB [STEALTH_MCP_CLONE_STORAGE_CAP_GB] browser-session cap : 20.0 GB [STEALTH_MCP_BROWSER_SESSION_STORAGE_CAP_GB] diff --git a/audit/stage2/finding_F776_no_per_tool_transport_evidence.md b/audit/stage2/finding_F776_no_per_tool_transport_evidence.md new file mode 100644 index 0000000..0fda8d9 --- /dev/null +++ b/audit/stage2/finding_F776_no_per_tool_transport_evidence.md @@ -0,0 +1,85 @@ +# F-776 — almost no served tool has per-tool real-transport success evidence + +**Status: OPEN, narrowed once.** Opened by RELEASE-5 (W5) while generating +`RELEASE_CONTRACT.md`. Narrowed the same day by PR #52 +(`tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip`), +which qualifies `set_cookie`, `get_cookies` and `clear_cookies` — the first and, +at this SHA, only per-tool transport evidence in the tree. +Not a product defect: an **evidence** gap, and the reason the generated contract +qualifies the number of tools it does. +**Severity: HIGH for claims, none for behaviour** — nothing here says a tool is +broken. It says the gate cannot prove, per tool, that the user-visible outcome +happens over the transport the user actually uses. + +--- + +## The finding + +plan_RELEASE §2.5 defines `release-qualified-success` for a tool as a row naming +"the precise user outcome asserted, fully-qualified passing node, required +transport, fixture or site shape, and required OS cells", each present as +current-run success evidence — and then rules out the substitutes: + +> A schema/type/non-null assertion, `.fn`-only call, **representative journey**, +> error-only test, exemption, or characterization cannot satisfy a transport, +> site-shape, manual-QA, or cross-OS success claim. + +Applied to the tree at `audit/release-integration` (`ff35ae3`): + +| Evidence tier that exists | Where | Why it cannot qualify a tool | +|---|---|---| +| the real-stdio journey | `tests/test_e2e_transport.py::test_real_stdio_release_gate_journey` | it is *the* representative journey (§2.1), explicitly disqualified as per-tool evidence | +| in-process E2E | `tests/test_e2e_*.py`, the `E2E_COVERED` manifest | drives tools through the `.fn` seam — a `.fn`-only call cannot satisfy a transport claim | +| in-memory client | `tests/test_mcp_protocol_surface.py` | an in-memory FastMCP client, not the wire | +| **per-tool transport** | `tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip` | **this one qualifies** — `set_cookie`/`get_cookies`/`clear_cookies`, on Linux/X64 + Windows/X64 | + +So at this SHA exactly three of the served tools have per-tool transport success +evidence, and the `get_cookies` hard block turned out to be the visible tip of a +general gap rather than a lone exception: the same bar applied to every other +tool leaves it unqualified. The cookie node is also the template for closing the +rest — a dedicated collected node, one harness, an asserted user outcome. + +## What this does NOT mean — say this as loudly as the finding + +**`served-unqualified` is not `untested`.** Every served tool is driven against +**real Chrome** by the E2E suite, and a set-equality tripwire keeps that coverage +complete — at this SHA the manifest is 94 covered, 0 exempt. The transport lane +separately proves the wire path itself works on two OS cells. + +What is missing is narrower and more specific: a **per-tool** assertion of the +user-visible outcome **over stdio**. The gap is *where the tests run*, not whether +they run, and closing it is test placement rather than a product fix. + +A reader who collapses that distinction gets it wrong in one of two directions — +either dismissing the product ("91 untested tools") or dismissing the contract +("they're clearly tested, this document is pedantic"). Both are wrong; the +contract's §5 states each tool's actual evidence for exactly this reason. + +Also true: no served tool is in the *no execution evidence* state. If one ever +lands there it is materially weaker than the rest, and the generated table says +so in its own words rather than folding it into the same bucket. + +## What closing it requires + +Per-tool assertions in the `transport` lane (real launcher, real stdio, real +Chrome) that name a user outcome and assert it — exactly the shape PR #52 +landed for the cookie tools. Each such node, once it passes on the required +cells, is added to `tools/release_tool_claims.json` and the `release-evidence` +job re-verifies it against that run's ledger; the contract's count then moves on +its own, with no prose to update. + +Two bounds constrain any such claim before it is written: + +* `transport` runs on **Linux/X64 and Windows/X64 only** — macOS/ARM64 is excluded + under F-773, so a per-tool stdio claim can be qualified on **two** cells, never + three; +* a claim citing the representative journey node is rejected by the ledger, by + design (`NON_PER_TOOL_NODES`). + +## Routing + +Recorded here and in the generated contract's limitations register. It is a +plan-level scope question (how much per-tool transport coverage a release +requires), not something RELEASE-5 may fix by relabelling: W5 is forbidden from +`src/` edits, and adding per-tool transport tests is new acceptance surface that +belongs to a plan step the human authorizes. diff --git a/audit/stage2/finding_F777_get_cookies_fn_seam_hang.md b/audit/stage2/finding_F777_get_cookies_fn_seam_hang.md new file mode 100644 index 0000000..dc64ca4 --- /dev/null +++ b/audit/stage2/finding_F777_get_cookies_fn_seam_hang.md @@ -0,0 +1,62 @@ +# F-777 — `get_cookies` hangs through the in-process `.fn` seam and poisons the tab's CDP connection + +**Status: OPEN.** Opened by RELEASE-5 (W5) when the real-transport cookie node +(PR #52) cleared the `get_cookies` hard block and, in doing so, disproved the +reason the tool had been exempt. +**Severity: MEDIUM for the test harness, NONE for the served product** — the path +a user drives works; the path the E2E suite drives does not. + +--- + +## The finding + +`get_cookies` carried a standing `E2E_EXEMPT` entry whose stated reason was that +it "hangs against real Chrome … and poisons the tab's CDP connection so every +later call times out". Measured on the merged tree, same tool and same Chrome, +that reason is **seam-specific, not product-wide**: + +| Seam | `Network.getCookies` | `Network.getAllCookies` | next call on that tab | +|---|---|---|---| +| in-process `.fn` (what `tests/test_e2e_*.py` use) | hangs ~30s, never returns | hangs ~30s, never returns | dies with a 10s CDP timeout | +| real stdio transport (what a user gets) | returns correctly | returns correctly | unaffected | + +So the exemption was recording a **harness** limitation as if it were a product +defect. The tool now has a passing per-tool success assertion over the real +transport — `tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip`, +which asserts the retrieved VALUE — and `E2E_EXEMPT` is empty. + +## Why the blast radius is wider than one tool + +The failure is not "one call returns nothing". The tab's CDP connection is left +unusable, so the *next* call on that tab fails too. Any `.fn`-seam test that +touches `get_cookies` therefore corrupts whatever runs after it in the same tab, +and the symptom surfaces on the innocent test rather than on the cause. + +Practical rule, and the reason `tests/test_e2e_interaction.py::test_cookies_lifecycle` +asserts through `document.cookie` instead: **no `.fn`-seam test may call +`get_cookies`.** Its coverage lives in the transport lane. + +## What this does NOT mean + +* It is not a user-facing defect. Nothing a user does goes through the `.fn` + seam; the transport path is evidenced and qualified in `RELEASE_CONTRACT.md`. +* It does not mean the E2E suite is unsound. It means one tool is unreachable + from it, and the suite now says so instead of hiding it behind an exemption + whose reason was wrong. +* It is not the same finding as F-776. F-776 is that *most* tools lack per-tool + transport evidence; F-777 is that *this* tool cannot be reached in-process at + all. + +## What closing it requires + +A cause: whether the hang is in nodriver's `get_all_cookies`/`get_cookies` +wrappers, in how the in-process seam shares the CDP connection, or in the +absence of the transport's connection lifecycle. Until then the rule above is +the mitigation, and it is enforced by the coverage manifest's comment plus the +transport-lane placement. + +## Routing + +Recorded here and in the generated contract's limitations register. No `src/` +change is proposed: W5 is forbidden from production edits, and the served path is +correct — a fix here is test-infrastructure work that the human schedules. diff --git a/audit/stage2/finding_F778_get_cookies_return_type_mismatch.md b/audit/stage2/finding_F778_get_cookies_return_type_mismatch.md new file mode 100644 index 0000000..bbc5160 --- /dev/null +++ b/audit/stage2/finding_F778_get_cookies_return_type_mismatch.md @@ -0,0 +1,43 @@ +# F-778 — `get_cookies` returns nodriver `Cookie` dataclasses, not the declared `list[dict]` + +**Status: OPEN.** Opened by RELEASE-5 (W5) from the measurements the +real-transport cookie node (PR #52) produced. +**Severity: LOW / cosmetic** — the wire shape a client receives is correct. The +mismatch is between the annotation and the runtime object, plus one confusing +client-side reconstruction. + +--- + +## The finding + +`get_cookies` is declared `-> list[dict[str, Any]]` but returns nodriver +`cdp.network.Cookie` **dataclasses**. What each consumer sees: + +| Consumer | What it gets | Correct? | +|---|---|---| +| the MCP wire (`structuredContent`) | proper JSON cookie objects — pydantic serializes the dataclasses | **yes** | +| a client reading `structured_content` | real cookie dicts with `name`/`value`/`domain`/`path` | **yes** — this is what the qualified test asserts | +| fastmcp's `result.data` reconstruction | an opaque `[Root()]` | no, but it is a client-side artifact | +| the type annotation | claims `dict`, delivers a dataclass | no | + +No user impact has been measured: the served path returns the right data in the +right shape. The risk is one of *reading*, not of behaviour — an +`.data`-based assertion looks like a product failure when it is only the +reconstruction, which is why +`tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip` +deliberately asserts through `structured_content` and says so in its docstring. + +## What closing it requires + +Either the annotation matches what is returned, or the tool converts the +dataclasses to plain dicts before returning. Both are `src/` changes, so neither +belongs to W5 (production edits are a hard non-goal). Whichever is chosen, the +transport node above already pins the observable wire shape, so a change that +altered it would fail loudly. + +## Routing + +Recorded here and in the generated contract's limitations register, so the +mismatch lives somewhere rather than nowhere. Low priority by severity, but it +sits on a now-qualified tool, which is why it is written down rather than +carried in a review comment. diff --git a/audit/stage2/finding_F779_macos_integration_teardown_flake.md b/audit/stage2/finding_F779_macos_integration_teardown_flake.md new file mode 100644 index 0000000..674677f --- /dev/null +++ b/audit/stage2/finding_F779_macos_integration_teardown_flake.md @@ -0,0 +1,233 @@ +# F-779 — `integration (macOS/ARM64)` fails at teardown with `Event loop is closed`, then passes on an identical tree + +**Status:** open (gate reliability, not a user-facing product defect) +**Opened by:** the W5 contract re-run, 2026-07-28 +**Severity:** **HIGH.** Measured failure rate is roughly **one run in four** on the +macOS/ARM64 integration cell, and each failure reddens the aggregate +`release-gate` check — the one check a repository ruleset is meant to require. +A required check that fails a quarter of the time for reasons unrelated to the +change is not a gate; it is a coin flip with a retry button. + +> **Revision note (2026-07-29):** this finding originally said "observed once then +> not reproduced." That was wrong within hours. It has now reproduced on a +> **documentation-only commit**, which settles the question of whether code is +> involved. The rate below is measured, not estimated. + +## What was observed + +Commit `d13997e` (W5, "regenerate the contract at 2.0.0") failed CI with three +reds: + +| check | conclusion | +|---|---| +| `release-gate / integration (macOS/ARM64)` | failure | +| `release-gate / release-evidence` | failure (consequence) | +| `release-gate / release-gate` (aggregate) | failure (consequence) | + +Only the first is a root cause. The other two are the fail-closed machinery +working exactly as designed: `release_evidence` reported +`integration/macOS-ARM64: non-success terminal outcome 'failure'` and refused to +certify the ledger, and the aggregate reported `one or more required edges were +not success`. **Neither is a separate defect.** + +The macOS job's failure annotations were: + +``` +Process completed with exit code 1. (.github:243 — "Run integration + Chrome-identity tests") +Event loop is closed (.github:545) +Event loop is closed (.github:545) +``` + +## Why it is a flake and not a regression + +`6a8fa79` is an **empty commit on top of `d13997e`** — `git commit-tree` against +`d13997e^{tree}`, so the two commits have a **byte-identical tree**. The gate was +re-run against that identical tree and returned **32/32 success**, including +`integration (macOS/ARM64)`, `release-evidence`, and the aggregate. + +Same code, same workflow, same runner image: one red, one green. That is the +definition of a flake. + +For completeness, `d13997e`'s only delta against the previously fully-green +`5c2505a` was version metadata (`1.2.0` → `2.0.0`) and `[tool.hatch.build.targets.sdist]` +excludes — packaging-only changes that `d13997e`'s own **six green install-smoke +cells** already exercised. There was never a plausible mechanism by which that +diff could break a macOS integration teardown. + +## Second observation — the one that settles it (2026-07-29) + +Commit `7c65374` adds **exactly one file**: `finding_F780_...md`, a markdown +document. No code, no workflow, no test, no dependency. It failed with the +byte-identical signature: + +``` +Process completed with exit code 1. (.github:241 — "Run integration + Chrome-identity tests") +Event loop is closed (.github:545) +Event loop is closed (.github:545) +``` + +A markdown file cannot break a macOS integration test. Combined with the +identical-tree re-run above, the code is exonerated twice over by two independent +methods. + +### Measured rate on the W5 line + +| commit | what changed | `integration (macOS/ARM64)` | +|---|---|---| +| `5028d66` | flake record | success | +| `5c2505a` | 2.0.0 merge | success | +| `d13997e` | contract regen | **failure** | +| `6a8fa79` | *empty commit, identical tree* | success | +| `a97c970` | F-779 doc + uv.lock one-liner | success | +| `7c65374` | **one markdown file** | **failure** | +| `7a3f546` | F-779 rate correction (docs) | success | +| `3be448b` | F-779 mechanism (docs) | success | + +**2 failures in 8 consecutive runs (~25%), as of `3be448b`.** + +Both failures carry the same `Event loop is closed` teardown signature. Successes +bracket each failure, so this is not a regression that landed and stayed. + +> **On the number itself.** This tally is *live*, and it has a self-invalidating +> property worth naming: every commit that edits this file adds another sample to +> the denominator. It read 2/6 when first written, then 2/7, now 2/8 — not because +> anything changed, but because documenting it generates evidence about it. Do not +> keep re-editing the percentage; it will always be slightly behind. +> +> **The durable claim, which none of that drift touches:** the macOS/ARM64 +> integration cell fails intermittently at an order of roughly **one run in four**, +> the failures are provably independent of the code under test, and each one takes +> the `release-gate` aggregate down with it. Anyone tempted to update the +> fraction should instead spend that effort on the mechanism below. + +Note the compounding arithmetic: the aggregate needs *every* edge green, so a +~25% failure on one cell puts the headline `release-gate` check red roughly one +run in four no matter how healthy the other 31 jobs are — and that is a floor, +since the Linux cold-spawn flake can independently redden the same aggregate. + +## Probable mechanism — HYPOTHESIS, not a confirmed diagnosis + +**Confidence: moderate-to-high on the mechanism, zero direct confirmation.** No +macOS machine was available and job logs require admin rights, so nothing below +was observed — it is derived from reading the teardown path. Treat it as the +first place to look, not as the answer. + +`browser_manager.close_instance` Phase 3 (`browser_manager.py:923-940`): + +```python +stop_coro = await asyncio.wait_for( + asyncio.to_thread(self._blocking_teardown, instance_id, browser), + timeout=self.CLOSE_KILL_TIMEOUT, # settings default: 5.0s +) +except TimeoutError: + ... "worker thread continues in background, orphan will be reaped by process_cleanup" +``` + +The comment is correct and the code knows it: **`asyncio.wait_for` cancels the +awaitable, but it cannot cancel the worker thread.** `asyncio.to_thread` dispatches +to a `ThreadPoolExecutor`; on timeout the coroutine gives up while the thread keeps +running. That leaves a thread alive, holding `browser`, after `close_instance` has +returned and bookkeeping has moved on. + +Two ways that thread produces exactly `Event loop is closed` once the loop is gone: + +1. **Future resolution.** When the orphaned thread finishes, the executor resolves + its future via `loop.call_soon_threadsafe(...)`. Against a closed loop that is + `RuntimeError: Event loop is closed`, raised from the thread, outside anyone's + `try`. +2. **`browser._process.terminate()` (`browser_manager.py:238`).** nodriver's + `_process` is an **asyncio** subprocess bound to the loop that created it. The + retry loop calls `.terminate()` / `.kill()` from the worker thread; once that + loop is closed, the transport's `_check_closed()` raises + `RuntimeError: Event loop is closed`. The surrounding `except Exception` then + falls through to `.kill()`, which fails the same way — which is a plausible + reading of why the annotation shows the message **twice**. + +Why this fits F-779's fingerprint specifically: + +- **Intermittent** — only bites when kill exceeds the 5s `CLOSE_KILL_TIMEOUT`. +- **Teardown-only** — nothing before teardown touches this path. +- **Code-independent** — any commit can lose the race, which is why a + markdown-only commit hit it. +- **macOS-leaning** — macOS is already the anomalous cell here (F-773: Chrome under + the detached backend completes no network navigation on that runner). A cell where + Chrome/process behaviour is already known to differ is exactly where a 5s kill + budget would be tightest. + +### How to confirm it cheaply + +Get one admin-authenticated job log and check whether the `RuntimeError` traceback +originates in a thread (`ThreadPoolExecutor-N_M`) rather than the main task. If it +does, the hypothesis holds. If the traceback is in the main task, this section is +wrong and should be deleted rather than argued for. + +### Note for whoever fixes it + +The tempting fix — widen `CLOSE_KILL_TIMEOUT` — only moves the race. The structural +issue is that a thread which cannot be cancelled is abandoned while still holding +loop-bound objects. Options worth weighing: join the orphaned thread at shutdown, +make `_blocking_teardown` touch only OS-level primitives (`os.kill`, psutil) and +never the asyncio `Process`, or keep a registry of abandoned threads that shutdown +drains. **This is a `src/` change and plan_RELEASE forbids `src/` edits, so it +belongs to a FIX plan, not here.** + +## What is NOT yet known + +The mechanism is not diagnosed. `Event loop is closed` at teardown is consistent +with an asyncio loop being closed while a task or transport still holds a +reference to it, but the specific owner was not identified, because: + +- **Job logs require admin.** The unauthenticated REST API returns `403 Must have + admin rights to Repository` for `/actions/jobs//logs`. Check-run + **annotations** were the only window, and they carry the message but not the + traceback. +- The failure was not reproduced locally (this is a macOS-only observation and no + Mac is available in this environment). + +So this finding records *that* it flakes and *that* the tree is exonerated. It +does **not** claim to know why. + +## Relationship to neighbouring findings + +- **Not F-773.** F-773 is "macOS/ARM64 Chrome under the detached backend completes + no network navigation on hosted runners" — reproducible 11/11, and the reason + the gate makes no macOS navigation claim. F-779 is a *teardown-time* error on a + job that otherwise ran, and it is *not* reproducible. +- **Not the Linux cold-spawn flake.** That one is a cold-*start* race + (`Failed to connect to browser`) on a different OS at a different phase. Same + category (gate reliability), different mechanism. +- **Same category as F-775b's macOS close-flake observation** — an unreproducible + macOS teardown/close anomaly. Worth checking whether they share a root cause; + that has not been done. + +## Why it matters + +`plan_RELEASE` §0.2 makes flake-freedom one of the three properties behind +"green ⇒ blindly pushable". This is now the **second** distinct gate flake on +record (after the Linux cold-spawn one), and unlike that one it takes down the +**aggregate check itself**. + +The practical hazard is cultural, not technical: a required check that +intermittently reddens for reasons unrelated to the change trains reviewers to +re-run until green, which is indistinguishable from training them to ignore it. +That is precisely the failure mode this campaign exists to prevent. + +## Disposition + +- Recorded as a limitation in the generated release contract (gate reliability). +- **Does not block the 2.0.0 contract on correctness** — the tree is green at + `6a8fa79` and `a97c970`, and the failures are provably code-independent. +- **It does block the "green ⇒ blindly pushable" claim**, and that claim is the + stated point of the whole campaign. A gate whose headline check is red ~1 run in + 3 for unrelated reasons cannot license a blind push, because the reviewer can no + longer distinguish "my change is bad" from "the gate did the thing it does." + Whether to tag 2.0.0 anyway is a human call; what is not available is calling + the gate trustworthy while this is open. +- Correct owner is **W8** (flake quarantine), which has not run. W8 should treat + "identical tree, different conclusion" as its acceptance criterion rather than + a retry budget. +- If it recurs, capture the job log with an admin-authenticated `gh` (unavailable + in the session that opened this) and attach the traceback here before + attempting a fix. **Do not "fix" this by adding a retry** — a retry that hides + a teardown bug is exactly the second-way-to-do-something defect the repo's + conventions forbid. diff --git a/audit/stage2/finding_F780_legacy_test_yml_timeout_below_harness_bounds.md b/audit/stage2/finding_F780_legacy_test_yml_timeout_below_harness_bounds.md new file mode 100644 index 0000000..a06baf0 --- /dev/null +++ b/audit/stage2/finding_F780_legacy_test_yml_timeout_below_harness_bounds.md @@ -0,0 +1,101 @@ +# F-780 — the legacy `test.yml` browser-integration job cannot pass the W1 transport journey: its `--timeout=120` is *below* the harness's own inner bounds + +**Status:** open, but **self-resolving** — the offending job is deleted by W2 (#44). +**Opened by:** merge-queue triage, 2026-07-28. +**Severity:** low as a defect, **high as a trap.** It makes three PRs in the release +stack look like they carry a product regression when they do not. + +## Symptom + +Three open PRs in the release stack are red on a check named +`Browser Integration Tests (Chrome + Xvfb)`: + +| PR | branch | legacy job | +|---|---|---| +| #43 | `audit/release-fix-b` | **failure** | +| #45 | `audit/release-4-w4` | **failure** | +| #47 | `audit/release-fix-d` | **failure** | + +The same check is **green** on `main` (~3 min) and on **#42** (`audit/release-1-w1`, +also ~3 min). So it looks exactly like "FIX-B broke the browser integration tests." + +**It did not.** + +## Root cause + +`.github/workflows/test.yml` (the pre-W2 workflow) runs: + +```yaml +uv run pytest -m integration -v --tb=short --timeout=120 +``` + +`tests/release_gate_harness.py` declares its own per-step bounds, under a comment +that states the intent outright — *"every await is wrapped; the pytest --timeout is +the outer net"*: + +```python +INIT_TIMEOUT = 60.0 +LIST_TIMEOUT = 130.0 # first backend-bound call — covers backend cold start +SPAWN_TIMEOUT = 120.0 # first real Chrome launch +WARMUP_TIMEOUT = 150.0 # cold Chrome + master-profile bootstrap +WARMUP_ATTEMPTS = 4 # with 3s * attempt backoff +``` + +`LIST_TIMEOUT` (130s) and `WARMUP_TIMEOUT` (150s) are each **larger than the whole +job's 120s per-test budget**, and `BACKEND_READY_TIMEOUT = 120.0` in +`embedded/singleton.py` is exactly equal to it. A 120s outer net cannot contain a +single inner step budgeted at 130s or 150s. The job is **structurally incapable** +of running this test to completion — no timing luck involved. + +## Why it started failing exactly at FIX-B + +W1 landed the transport journey **already marked `xfail(strict=False)`**, because +B1 (per-MCP-session `app_lifespan` + the proxy's 2s watchdog) was a known open +defect at the time. An xfail costs nothing on the clock, so the legacy job stayed +green on #42. + +FIX-B's C2 commit (`585ebf2`, "flip the W1 transport xfail — journey green") removed +that marker, which was correct: FIX-B fixed B1, so the test must really run. From +that commit onward the legacy job actually executes the journey, hits its 120s wall, +and fails. + +#45 (W4) and #47 (FIX-D) branch off FIX-B, so they inherit both the un-xfailed test +and the under-budgeted workflow. + +## Why the release gate is green on the same code + +W2 (#44) replaced the legacy job's semantics with the reusable `release-gate` +workflow, which budgets realistically: + +- `integration` job: `--timeout=180` +- dedicated `transport` job: `--timeout=300` + +#46 (`audit/release-fix-c`) — which contains FIX-B's commits transitively — is +**green 23/23**, including `integration (Linux/X64)` running the *same* +`-m integration` selection. That is the controlled comparison: same code, same OS, +same marker, different budget, different outcome. + +## Disposition + +**Not fixed here, deliberately.** The job disappears when #44 merges, so patching +`test.yml` on three separate in-review branches would be churn against a file that +is about to be deleted — and it would mutate PRs while a human is reviewing them. + +**What the human needs to know when working the merge queue:** merging in stack +order will show red at #43 → #45 → #47 until **#44 (W2)** lands, at which point the +legacy job no longer exists and the release gate takes over. Those reds are not a +signal about the code. The signal that matters is the tip of the stack, which was +green 32/32. + +**Do not "fix" this by re-adding the xfail.** The xfail was correct only while B1 +was open; restoring it to quiet a workflow that is being deleted would re-hide a +defect that is genuinely fixed — and would make the transport journey stop proving +the one thing it exists to prove. + +## Wider lesson + +An outer timeout that is *smaller* than the inner bounds it is supposed to contain +is not a conservative setting — it is a broken one, and it converts into a +false product-regression signal at the exact moment a test stops being skipped. +Any workflow that runs this harness must budget above `WARMUP_TIMEOUT` (150s) plus +the warmup retry envelope, not below it. diff --git a/pyproject.toml b/pyproject.toml index a7e5324..29a15ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "stealth-chrome-devtools-mcp" -version = "1.2.0" +version = "2.0.0" description = "Undetectable browser automation for AI agents via the Model Context Protocol." readme = "README.md" requires-python = ">=3.11" @@ -94,6 +94,24 @@ source = ["src/stealth_chrome_devtools_mcp"] [tool.coverage.report] show_missing = true +[tool.hatch.build.targets.sdist] +# The sdist was 14.1 MB against a 191 KB wheel: 12.8 MB of demo media and 2.8 MB +# of internal audit plans, none of which is needed to build, install, test, or +# understand the package. W3's install-smoke installs the sdist on three runner +# cells, so the weight was paid on every gate run as well as by every user who +# builds from source. +# +# Only `pyproject.toml`, `README.md` and the embedded/js package data are +# structurally required (tools/package_verify.py `_check_sdist_members` / +# `_check_package_data`); `tests/` is kept deliberately so a downstream packager +# can still run the suite from the sdist. +exclude = [ + "assets", # demo media, referenced from the README by relative path + "audit", # the audit campaign's plans and findings — repo history, not a distribution + ".claude", # agent config and worktrees + ".husky", +] + [tool.hatch.build.targets.wheel] packages = ["src/stealth_chrome_devtools_mcp"] # NOTE: do not add a force-include for embedded/js here. Those files are tracked diff --git a/tests/MANUAL_QA_PROTOCOL.md b/tests/MANUAL_QA_PROTOCOL.md index eb54ee0..f4ceb57 100644 --- a/tests/MANUAL_QA_PROTOCOL.md +++ b/tests/MANUAL_QA_PROTOCOL.md @@ -489,14 +489,30 @@ ID belongs to the tab set, not that it is the focused target. **Evidence**: satisfied — pytest: `tests/test_e2e_interaction.py::test_cookies_lifecycle`. -### MQ-53: Get cookies — returns set cookies -**Manual**: call `get_cookies` after setting one. -**Evidence**: blocked — `[KNOWN-BUG: get_cookies_hang]` prevents successful -real-Chrome cookie retrieval; HEAD has no success-path acceptance target. +### MQ-53: Get cookies — returns the exact value that was set +**Rewritten by W5** (plan_RELEASE §2.5 option (a) — the hard block cleared). The +former step was `blocked` on a `[KNOWN-BUG: get_cookies_hang]` that, measured, +belongs to the in-process `.fn` seam and not to the served path (F-777): over +real stdio against real Chrome, retrieval works and is now asserted. + +**Manual**: `set_cookie` a value on a real `http://` origin, then `get_cookies` +and compare the returned value **byte for byte** with what you set. Presence, a +non-empty list, or a type check is not this step; the VALUE is. Then +`clear_cookies` and re-read to confirm it is gone. +**Evidence**: satisfied — pytest: +`tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip`. + +Bounds this step does **not** exceed: the node runs in the `transport` lane, so +its evidence is **Linux/X64 and Windows/X64 only** — macOS/ARM64 is excluded +under F-773, and no macOS cookie claim exists. It is a dedicated node, never the +representative journey (§2.5 disqualifies that as per-tool evidence), and it is +the row backing `get_cookies` in `tools/release_tool_claims.json`, which the +`release-evidence` job re-verifies against every run's records. **Current support (non-acceptance)**: pytest: `tests/test_e2e_functions_hooks.py::test_e2e_coverage_manifest` records the -exemption only. Schema and missing-instance checks are not behavioral coverage, -and no fake unit success may clear this step. +covered/exempt partition only — `get_cookies` moved into `E2E_COVERED` when the +node above landed, and `E2E_EXEMPT` is now empty. Membership in that manifest is +an inventory fact; it never converted, and cannot convert, into a success claim. ### MQ-54: Clear cookies **Manual**: `clear_cookies` → `get_cookies` (or `execute_script` reading @@ -1007,14 +1023,29 @@ Chrome identity, and zero skipped/xfail/failed required nodes. ## Phase 18 — Tool Coverage Completeness -### MQ-111: Every advertised tool has ≥1 E2E test -**Manual**: compare `tools/list` output to the test manifest; no tool is untested. -**Evidence**: blocked — MQ-53 leaves `get_cookies` without successful behavioral -E2E coverage, so the every-tool claim is false at HEAD. -**Current support (non-acceptance)**: pytest: -`tests/test_e2e_functions_hooks.py::test_e2e_coverage_manifest` proves a -93-covered/one-exempt partition. The inventory node cannot convert its sole -`get_cookies` exemption into coverage. +### MQ-111: Every advertised tool has a visible state in the release contract +**Rewritten by W5** (plan_RELEASE §2.5 — "these states are never inferred from +F-108 set equality"). "Every tool has ≥1 E2E test" was an *inventory* +requirement: set equality over a coverage manifest, which proves membership and +not behaviour. W5 replaces it with the requirement a reader can act on — every +served tool carries a **state**, and every claimed success is backed by ledger +evidence that the run produced. + +**Manual**: compare `tools/list` output with `RELEASE_CONTRACT.md` §5. Every +served tool appears exactly once with a state; every `release-qualified-success` +row names a passing node, a transport, a site shape and its OS cells; every other +row carries a tracking id and a user impact. No tool is silently absent, and no +tool is qualified by exemption, by counting, or by membership in a manifest. +**Evidence**: satisfied — pytest: +`tests/test_release_contract.py::test_the_tool_table_covers_every_served_tool_exactly_once`. +**Current support (non-acceptance)**: pytest: +`tests/test_release_contract.py::test_every_served_unqualified_row_carries_a_tracking_id_and_impact` +proves the tracking-id/impact half, and +`tests/test_e2e_functions_hooks.py::test_e2e_coverage_manifest` still proves the +covered/exempt partition — an evidence *tier*, never a qualification. The number +of release-qualified tools is derived from the claim ledger and re-verified +against each run's records; every tool without its own per-tool transport +assertion is bounded by F-776, not by this step. ### MQ-112: Every advertised tool has ≥1 transport-tier test OR explicit exemption **Manual**: same check through the real transport layer. diff --git a/tests/release_gate_harness.py b/tests/release_gate_harness.py index 3808ed9..a4c31d3 100644 --- a/tests/release_gate_harness.py +++ b/tests/release_gate_harness.py @@ -60,6 +60,7 @@ import tempfile import threading import time +import uuid from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any @@ -72,10 +73,13 @@ _log = logging.getLogger("release_gate_harness") # ── Contract constants ────────────────────────────────────────────────────── -# How far `run_release_gate_journey` runs. ONE journey, two declared extents — -# never two journeys. See that function's docstring for what each one claims. +# Which declared segment of the ONE gate mechanism `run_release_gate_journey` +# runs. These are extents/segments, never duplicated machinery: every value +# below shares the same launcher resolution, isolated env, fixture app, stdio +# client, and teardown. See that function's docstring for what each claims. FULL_JOURNEY = "full" HANDSHAKE_ONLY = "handshake" +COOKIE_ROUND_TRIP = "cookies" SERVER_NAME = "stealth-chrome-devtools-mcp" REGISTRY_TOOL_COUNT = 94 # remediation baseline (CLAUDE.md: derived == 94) @@ -758,6 +762,128 @@ async def _cold_start_warmup( ) +async def _cookie_round_trip( + client: Client, iid: str, page_url: str, journey: dict[str, Any] +) -> None: + """plan_RELEASE W5 — the real success path for ``get_cookies``. + + W5's ``get_cookies`` hard block (§2.5) requires a real Chrome + real + transport test that **sets a cookie, retrieves it, and asserts its value**; + a schema check, a mock, a missing-instance error, or a characterization + explicitly cannot satisfy it. This does the full round trip through + ``tools/call`` only: + + ``set_cookie`` → ``get_cookies(urls=[page_url])`` → **assert the value** → + ``get_cookies()`` (the no-argument default, a different CDP call) → + ``document.cookie`` cross-check → ``clear_cookies`` → assert it is gone. + + The value is unique per run, so a stale cookie from an earlier run in a + reused profile cannot forge the assertion, and the ``document.cookie`` + cross-check proves the cookie really reached the browser rather than + ``get_cookies`` merely echoing what ``set_cookie`` was handed. + + ``get_cookies`` is declared ``-> list[dict[str, Any]]`` but returns nodriver + ``cdp.network.Cookie`` **dataclasses**. pydantic serializes those to correct + JSON objects, so ``structuredContent`` — what ``_call`` returns, and what a + user's client receives — is a list of cookie dicts. Asserting through + fastmcp's ``result.data`` instead would see an opaque ``[Root()]``; that is + a property of the reconstruction, not a product defect. + """ + name = "release_gate_cookie" + value = f"w5-{uuid.uuid4().hex}" + cookies: dict[str, Any] = {"name": name, "value": value} + journey["cookies"] = cookies + + assert await _call( + client, + "set_cookie", + {"instance_id": iid, "name": name, "value": value, "url": page_url}, + CALL_TIMEOUT, + ), "set_cookie did not report success" + + # Retrieval #1 — scoped to the page URL (CDP Network.getCookies). + scoped = await _call( + client, "get_cookies", {"instance_id": iid, "urls": [page_url]}, CALL_TIMEOUT + ) + assert isinstance(scoped, list) and scoped, ( + f"get_cookies(urls=…) returned {scoped!r}" + ) + assert all(isinstance(c, dict) for c in scoped), ( + f"get_cookies did not serialize to dicts: {[type(c).__name__ for c in scoped]}" + ) + scoped_hit = next((c for c in scoped if c.get("name") == name), None) + assert scoped_hit is not None, f"{name!r} absent from get_cookies(urls=…): {scoped}" + # THE assertion W5's hard block turns on. + assert scoped_hit.get("value") == value, (scoped_hit.get("value"), value) + cookies["scoped_value"] = scoped_hit["value"] + cookies["scoped_count"] = len(scoped) + cookies["field_names"] = sorted(scoped_hit) + + # Retrieval #2 — the no-argument default (CDP Network.getAllCookies). + every = await _call(client, "get_cookies", {"instance_id": iid}, CALL_TIMEOUT) + assert isinstance(every, list), f"get_cookies() returned {every!r}" + all_hit = next( + (c for c in every if isinstance(c, dict) and c.get("name") == name), None + ) + assert all_hit is not None, f"{name!r} absent from get_cookies(): {every}" + assert all_hit.get("value") == value, (all_hit.get("value"), value) + cookies["all_value"] = all_hit["value"] + + # Ground truth: the cookie is really in the browser, not just in our reply. + document_cookie = await _eval(client, iid, "document.cookie") + assert f"{name}={value}" in str(document_cookie), document_cookie + cookies["document_cookie_confirms"] = True + + # clear_cookies is part of the same real path — prove removal by re-reading, + # not by trusting its return value. + assert await _call( + client, "clear_cookies", {"instance_id": iid, "url": page_url}, CALL_TIMEOUT + ), "clear_cookies did not report success" + after = await _call( + client, "get_cookies", {"instance_id": iid, "urls": [page_url]}, CALL_TIMEOUT + ) + assert isinstance(after, list), after + assert not any(isinstance(c, dict) and c.get("name") == name for c in after), ( + f"{name!r} survived clear_cookies: {after}" + ) + cookies["cleared"] = True + + +async def _cookie_journey( + client: Client, base_url: str, record: dict[str, Any] +) -> None: + """W5's segment: spawn → navigate → cookie round trip → close. + + Deliberately minimal — everything here exists to make the cookie assertions + meaningful, so a failure reads as a cookie failure rather than as "the + journey" failing. A real page (not ``about:blank``) is required because + cookies need a real http:// origin, and the fixture app is that origin. + """ + spawn = await _call( + client, "spawn_browser", _headless_spawn_kwargs(), SPAWN_TIMEOUT + ) + assert isinstance(spawn, dict) and spawn.get("instance_id"), spawn + iid = spawn["instance_id"] + journey: dict[str, Any] = {"instance_id": iid} + record["journey"] = journey + try: + url = f"{base_url}/interact.html" + await _call(client, "navigate", {"instance_id": iid, "url": url}, NAV_TIMEOUT) + journey["navigated_url"] = url + await _settle_dom(client, iid) + + await _cookie_round_trip(client, iid, url, journey) + finally: + await _call( + client, + "close_instance", + {"instance_id": iid}, + CLOSE_TIMEOUT, + raise_on_error=False, + allow_fail=True, + ) + + async def _canonical_journey( client: Client, base_url: str, record: dict[str, Any] ) -> None: @@ -888,8 +1014,9 @@ async def run_release_gate_journey( ``work_dir`` is a throwaway directory (e.g. pytest ``tmp_path``) used for the isolated HOME (singleton state), session root, clone output, and logs. - ``stages`` selects how far the ONE journey runs — it never selects a - different journey: + ``stages`` selects which declared segment runs on top of the ONE shared + mechanism (launcher resolution, isolated env, fixture app, stdio client, + teardown). It never duplicates that machinery: ``"full"`` (default) everything: handshake, registry, parity, cold-start warmup, and the @@ -904,11 +1031,19 @@ async def run_release_gate_journey( — and the result record says so in ``stages`` so no consumer can read it as the full journey. Not an xfail: nothing failing is being marked as expected-to-fail; a smaller thing is being run and labelled. + ``"cookies"`` + the prefix plus warmup, then the focused ``set_cookie`` → + ``get_cookies`` → ``clear_cookies`` round trip (``_cookie_journey``) + instead of the canonical journey. plan_RELEASE §2.5 rules that a + *representative journey* cannot carry a per-tool success claim, so W5's + ``get_cookies`` evidence needs its own collected node + (``tests/test_e2e_transport_cookies.py``); this segment is what that + node drives. It navigates, so ``navigation_verified`` is true, but it + makes no canonical-journey claim. """ - if stages not in (FULL_JOURNEY, HANDSHAKE_ONLY): - raise ValueError( - f"stages must be {FULL_JOURNEY!r} or {HANDSHAKE_ONLY!r}, got {stages!r}" - ) + valid_stages = (FULL_JOURNEY, HANDSHAKE_ONLY, COOKIE_ROUND_TRIP) + if stages not in valid_stages: + raise ValueError(f"stages must be one of {valid_stages!r}, got {stages!r}") launcher = Path(launcher) work_dir = Path(work_dir) home_dir = work_dir / "home" @@ -929,7 +1064,7 @@ async def run_release_gate_journey( record: dict[str, Any] = { "schema_version": RESULT_SCHEMA_VERSION, "stages": stages, - "navigation_verified": stages == FULL_JOURNEY, + "navigation_verified": stages in (FULL_JOURNEY, COOKIE_ROUND_TRIP), "transport": "stdio", "launcher": str(launcher.resolve()), "singleton_port": port, @@ -960,9 +1095,12 @@ async def run_release_gate_journey( async with Client(transport, init_timeout=INIT_TIMEOUT) as client: await _foundation_proof(client, record) await _representative_parity(client, record) - if stages == FULL_JOURNEY: + if stages in (FULL_JOURNEY, COOKIE_ROUND_TRIP): await _cold_start_warmup(client, base_url, log_dir, record) + if stages == FULL_JOURNEY: await _canonical_journey(client, base_url, record) + elif stages == COOKIE_ROUND_TRIP: + await _cookie_journey(client, base_url, record) except BaseException as exc: # noqa: BLE001 PERMANENT(augment with child stderr + boot log, then re-raise) err = exc child_stderr = cap["text"] diff --git a/tests/test_e2e_functions_hooks.py b/tests/test_e2e_functions_hooks.py index 33dfd51..f8bea5c 100644 --- a/tests/test_e2e_functions_hooks.py +++ b/tests/test_e2e_functions_hooks.py @@ -358,9 +358,19 @@ def test_hook_doc_tools(): "execute_script", "get_page_content", "take_screenshot", - # cookies-storage (2 of 3; get_cookies is exempt below) — test_e2e_interaction + # cookies-storage (3) — set_cookie/clear_cookies: test_e2e_interaction.py. + # get_cookies: tests/test_e2e_transport_cookies.py, which sets a cookie, + # retrieves it, and asserts its VALUE over real Chrome + real stdio + # (plan_RELEASE W5's option (a)). It is covered there rather than here + # because it is reachable only over the transport: through the in-process + # `.fn` seam these E2E modules use, Network.getCookies/getAllCookies never + # returns and poisons the tab's CDP connection. Same tool, same Chrome — + # only the seam differs, so this is a harness limitation, not a gap in the + # product's user-facing path. Routed as a finding; do NOT call get_cookies + # from a `.fn`-seam test (see test_e2e_interaction.test_cookies_lifecycle). "set_cookie", "clear_cookies", + "get_cookies", # tabs (5) — test_e2e_interaction.py "list_tabs", "switch_tab", @@ -443,16 +453,16 @@ def test_hook_doc_tools(): } # Tools intentionally left to another tier, each with a reason. -E2E_EXEMPT: dict[str, str] = { - "get_cookies": ( - "hangs against real Chrome — the CDP Network.getCookies/getAllCookies " - "command never returns in the installed nodriver (get_all_cookies is " - "deprecated since 1.3) and poisons the tab's CDP connection so every " - "later call times out. Exercising it in E2E would risk leaking a Chrome " - "tree; the finding is reported for routing. set_cookie and clear_cookies " - "ARE E2E-covered (asserted via document.cookie)." - ), -} +# +# Empty as of plan_RELEASE W5-prep: `get_cookies` was the sole exemption, on the +# grounds that it "hangs against real Chrome". That reason turned out to be +# seam-specific rather than product-wide — it hangs through the in-process `.fn` +# seam, but over the real stdio transport it sets, retrieves, and clears cookies +# correctly (tests/test_e2e_transport_cookies.py asserts the retrieved VALUE). +# So it moved to E2E_COVERED and nothing remains exempt. Keep this dict: it is +# half of the partition tripwire below, and a future exemption belongs here WITH +# a reason, never as a silent deletion from E2E_COVERED. +E2E_EXEMPT: dict[str, str] = {} def test_e2e_coverage_manifest(): diff --git a/tests/test_e2e_interaction.py b/tests/test_e2e_interaction.py index 6f97ac3..c17eccb 100644 --- a/tests/test_e2e_interaction.py +++ b/tests/test_e2e_interaction.py @@ -339,11 +339,17 @@ async def test_get_element_state_pins_current_shape(fixture_app_server): async def test_cookies_lifecycle(fixture_app_server): """set_cookie + clear_cookies, verified via the live ``document.cookie``. - get_cookies is deliberately NOT called here — it hangs (the CDP - Network.getCookies / deprecated getAllCookies never returns in the installed - nodriver) and, worse, poisons the tab's CDP connection so every subsequent - call times out. It is E2E_EXEMPT with that finding; cookies are read via - document.cookie (non-httpOnly) instead, which is exact ground truth. + get_cookies is deliberately NOT called here — through the in-process ``.fn`` + seam this module uses, the CDP Network.getCookies / deprecated + getAllCookies never returns and, worse, poisons the tab's CDP connection so + every subsequent call times out. Cookies are read via document.cookie + (non-httpOnly) instead, which is exact ground truth. + + That hang is specific to this seam, NOT to the product: over the real stdio + transport the same tool against the same Chrome sets, retrieves, and clears + cookies correctly. ``tests/test_e2e_transport_cookies.py`` is where + get_cookies is covered (plan_RELEASE W5 requires the retrieved value be + asserted), which is why it is E2E_COVERED rather than E2E_EXEMPT. """ base = fixture_app_server spawn = get_fn("spawn_browser") diff --git a/tests/test_e2e_transport_cookies.py b/tests/test_e2e_transport_cookies.py new file mode 100644 index 0000000..bba64d9 --- /dev/null +++ b/tests/test_e2e_transport_cookies.py @@ -0,0 +1,105 @@ +"""plan_RELEASE W5-prep — the real-transport ``get_cookies`` success path. + +W5's **``get_cookies`` hard block** (plan_RELEASE §2.5): at W5 start, successful +real-browser cookie *retrieval* had never been proved, so the contract generator +refuses a 94-release-qualified-tool statement. The block clears only via option +(a) — "a real Chrome + real transport success test sets a cookie, retrieves it, +and asserts its value". A mock-only success, a missing-instance error, a schema +check, or a characterization explicitly **cannot** satisfy it. + +This module is that test, and it is a **dedicated collected node** on purpose. +§2.5 rules that a *representative journey* cannot satisfy a per-tool success +claim, and §2.1 names ``test_real_stdio_release_gate_journey`` as exactly that — +so folding these assertions into the canonical journey would have produced +evidence W5 must reject. The machinery is still the one harness (absolute +installed launcher, isolated HOME/session root, fixture app, stdio +``tools/call``, bounded teardown); only the steps differ. + +Covers three tools on the one real path — ``set_cookie``, ``get_cookies``, +``clear_cookies``. + +Marked ``integration`` + ``transport``; skipped when Chrome / the server is +unavailable (same guard as the other e2e modules). macOS transport is excluded +under F-773, so this node's evidence is Linux/X64 + Windows/X64. +""" + +from __future__ import annotations + +import shutil + +import pytest + +from e2e_helpers import CAN_RUN +from release_gate_harness import ( + COOKIE_ROUND_TRIP, + RESULT_SCHEMA_VERSION, + gate_work_dir, + resolve_launcher, + run_release_gate_journey, +) + +pytestmark = [pytest.mark.integration, pytest.mark.transport] + +if not CAN_RUN: + pytestmark.append(pytest.mark.skip("Chrome not available or server failed to load")) + + +async def test_real_transport_cookie_round_trip(tmp_path): + """set_cookie → get_cookies → **assert the value** → clear_cookies, over real + headless Chrome and real stdio JSON-RPC. + + Note the assertions read the harness record, which is built from + ``result.structured_content`` — the raw wire shape a user's client receives. + fastmcp's ``result.data`` reconstruction is deliberately NOT used: this tool + is declared ``-> list[dict[str, Any]]`` but returns nodriver + ``cdp.network.Cookie`` dataclasses, and while pydantic serializes those to + correct JSON objects on the wire, ``.data`` rebuilds them as an opaque + ``[Root()]``. Asserting through ``.data`` would look like a product failure + when it is only an artifact of the reconstruction. + """ + launcher = resolve_launcher() # this env's absolute installed console launcher + work_dir = gate_work_dir(tmp_path) # RUNNER_TEMP on CI (see helper docstring) + + try: + record = await run_release_gate_journey( + launcher=launcher, work_dir=work_dir, stages=COOKIE_ROUND_TRIP + ) + finally: + if work_dir != tmp_path: # pytest cleans its own; this one is ours + shutil.rmtree(work_dir, ignore_errors=True) + + assert record["schema_version"] == RESULT_SCHEMA_VERSION + assert record["transport"] == "stdio" + assert record["stages"] == COOKIE_ROUND_TRIP + assert record["navigation_verified"] is True # a real page, not about:blank + assert record["launcher"].endswith( + ("stealth-chrome-devtools-mcp.exe", "stealth-chrome-devtools-mcp") + ) + + journey = record["journey"] + assert journey["instance_id"] + # A real http:// origin, not about:blank — cookies need one. + assert journey["navigated_url"].startswith("http://127.0.0.1:") + + cookies = journey["cookies"] + expected = cookies["value"] + assert expected.startswith("w5-") # unique per run; a stale cookie cannot forge it + + # THE assertion the hard block turns on: the value we set came back, from + # both retrieval paths (Network.getCookies and Network.getAllCookies). + assert cookies["scoped_value"] == expected + assert cookies["all_value"] == expected + + # Retrieval returned real, populated cookie objects — not empty shells. + assert cookies["scoped_count"] >= 1 + assert {"name", "value", "domain", "path"} <= set(cookies["field_names"]) + + # The cookie really reached the browser; get_cookies did not merely echo us. + assert cookies["document_cookie_confirms"] is True + + # clear_cookies removed it (proved by re-reading, not by its return value). + assert cookies["cleared"] is True + + # Teardown left nothing running. + assert record["backend_gone"] is True + assert record["no_child_remaining"] is True diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py new file mode 100644 index 0000000..cb990cd --- /dev/null +++ b/tests/test_release_contract.py @@ -0,0 +1,366 @@ +"""The generated release contract may not overclaim (plan_RELEASE W5). + +`RELEASE_CONTRACT.md` is the document a reader trusts when they push blind, so +these tests police the two ways it could lie: + +* **drift** — the file on disk no longer matches what the generator produces + from the ledger, the claim ledger, and the live registry; +* **overclaim** — a number or sentence in it asserts more than the evidence + supports (a 94-qualified-tool statement, an unqualified "works on Linux, + Windows and macOS", an HTTP claim inherited from stdio evidence, an upgrade + claim W14 never made, or `get_cookies` presented as qualified). + +They are hermetic: they read the repository, never CI. +""" + +from __future__ import annotations + +import difflib +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools")) + +import gen_release_contract as gen # noqa: E402 PERMANENT(tools/ is not an importable package; the sys.path line above must run first) +import release_evidence as re_mod # noqa: E402 PERMANENT(tools/ is not an importable package; the sys.path line above must run first) + + +@pytest.fixture(scope="module") +def contract() -> str: + return gen.CONTRACT_PATH.read_text(encoding="utf-8") + + +def test_the_contract_is_regenerated_not_edited(): + """CI fails on drift: the document is output, never a hand-maintained file. + + The failure prints the first differing lines. A drift failure that only says + "stale" sends the reader to regenerate blindly, which is exactly how a + deliberate change gets rubber-stamped. + """ + problems = gen.check_contract() + if not problems: + return + diff = difflib.unified_diff( + gen.CONTRACT_PATH.read_text(encoding="utf-8").splitlines(), + gen.render_contract().splitlines(), + fromfile="RELEASE_CONTRACT.md (on disk)", + tofile="freshly generated", + lineterm="", + n=1, + ) + pytest.fail( + "RELEASE_CONTRACT.md is stale — regenerate it with " + "`uv run python tools/gen_release_contract.py --write` in the SAME " + "commit as whatever changed the ledger, claims, or registry.\n" + + "\n".join(list(diff)[:40]) + ) + + +def test_regeneration_is_reproducible(): + assert gen.render_contract() == gen.render_contract() + + +def test_the_tool_table_covers_every_served_tool_exactly_once(): + rows = gen.tool_rows() + names = [row.tool for row in rows] + assert sorted(names) == sorted(re_mod.registry_tool_names()) + assert len(set(names)) == len(names) + + +def test_every_served_unqualified_row_carries_a_tracking_id_and_impact(): + for row in gen.tool_rows(): + if row.state != "served-unqualified": + continue + assert row.tracking_id, f"{row.tool} has no tracking id" + assert row.impact, f"{row.tool} has no user impact" + + +def test_the_headline_count_is_derived_not_typed(contract: str): + counts = re_mod.tool_surface(re_mod.load_claims()) + served = counts["served_total"] + qualified = counts["release_qualified"] + assert f"qualifies {qualified} of the {served} served MCP tools" in contract + assert qualified + counts["served_unqualified"] == served + + +def test_no_ninety_four_qualified_tool_statement_exists(contract: str): + """The plan's hard block: 94 may not be claimed by exemption or by counting. + + A tool becomes qualified only through a verified claim row, so this test is + what makes the count fall out of evidence rather than out of a sentence. + """ + served = re_mod.tool_surface(re_mod.load_claims())["served_total"] + qualified = re_mod.tool_surface(re_mod.load_claims())["release_qualified"] + for phrase in ( + f"{served} release-qualified", + f"all {served} tools", + f"{served} qualified tools", + ): + assert phrase not in contract, f"contract overclaims: {phrase!r}" + if qualified != served: + assert f"qualifies {served} of the {served}" not in contract + + +def test_get_cookies_is_never_presented_as_qualified(contract: str): + """plan_RELEASE §2.5 option (b): the exclusion must be VISIBLE, not implied.""" + row = next(r for r in gen.tool_rows() if r.tool == "get_cookies") + claimed = {str(c.get("tool", "")) for c in re_mod.claim_rows(re_mod.load_claims())} + if "get_cookies" not in claimed: + assert row.state == "served-unqualified" + assert "get_cookies" in contract + assert row.tracking_id, "the excluded tool must carry a tracking id" + else: + assert row.state == "release-qualified-success" + assert "stdio" in row.tier + + +def test_every_qualified_claim_cites_a_node_that_exists_in_this_tree(): + """A claim for a phantom node is caught here, not three CI hours later. + + CI is the authority (the ledger re-verifies the node executed AND passed on + every cell claimed); this is the cheap local tripwire in front of it. + """ + for claim in re_mod.claim_rows(re_mod.load_claims()): + node = str(claim["node_id"]) + file_part, _, rest = node.partition("::") + path = gen.REPO_ROOT / file_part + assert path.is_file(), f"claim cites a file that does not exist: {node}" + func = rest.split("::")[-1].split("[")[0] + assert f"def {func}(" in path.read_text(encoding="utf-8"), ( + f"claim cites {node}, which {file_part} does not define" + ) + + +def test_qualified_claims_are_bounded_to_the_cells_that_can_evidence_them( + contract: str, +): + """A stdio claim is qualified on TWO cells — never three (F-773).""" + transport_cells = { + spec.key for spec in gen.matrix_rows() if spec.job == "transport" + } + assert len(transport_cells) == 2 + for claim in re_mod.claim_rows(re_mod.load_claims()): + if claim.get("transport") != "stdio": + continue + cells = set(map(str, claim["required_cells"])) + assert cells <= transport_cells, ( + f"{claim['tool']} claims stdio on a cell that does not run the " + f"real-stdio lane: {sorted(cells - transport_cells)}" + ) + if any( + c.get("transport") == "stdio" for c in re_mod.claim_rows(re_mod.load_claims()) + ): + assert "qualified on exactly two" in contract, ( + "the two-cell bound must be stated, not left for the reader to infer" + ) + + +def test_served_unqualified_never_reads_as_untested(contract: str): + """Over-qualification is dishonest in the other direction. + + "3 qualified / 91 served-unqualified" would be read as "91 untested tools", + which is false: those tools ARE driven against real Chrome. The contract has + to make a skeptical reader and a fair reader arrive at the same + understanding — tested, but not proved at the wire. + """ + assert "does not say" in contract or "does *not* say" in contract + assert "tools are untested" in contract + assert "not proved at the wire" in contract + assert "real Chrome" in contract + for row in gen.tool_rows(): + if row.state != "served-unqualified": + continue + assert row.tier, f"{row.tool} has no stated evidence" + assert row.tier != "served-unqualified", ( + "the evidence column must say what evidence EXISTS, not repeat the label" + ) + + +def test_a_tool_with_no_execution_evidence_is_called_out_separately(contract: str): + """The weakest bucket may never hide inside the general one.""" + uncovered = [r.tool for r in gen.tool_rows() if "no execution evidence" in r.tier] + if uncovered: + assert "**no execution evidence**" in contract, ( + f"{uncovered} have no execution evidence and the contract does not " + f"distinguish them" + ) + else: + assert "if none appears below, no served tool is in that state" in contract + + +def test_the_contract_does_not_claim_flake_freedom(contract: str): + """§0.2 makes flake-freedom load-bearing; the gate has an observed flake.""" + assert "does **not** claim the gate is flake-free" in contract + assert "Linux cold-spawn flake" in contract + # The flake has hit a cell a qualified claim depends on; saying so is the + # whole point of the row. A future edit that softens it back to a + # cosmetic-cell-only story must fail here. + assert "transport (Linux/X64)" in contract + + +def test_macos_transport_is_named_as_excluded_not_covered(contract: str): + assert "F-773" in contract + assert "excluded" in contract.lower() + transport_cells = { + spec.label for spec in gen.matrix_rows() if spec.job == "transport" + } + assert "macOS/ARM64" not in transport_cells + + +def test_the_os_family_claim_is_always_qualified(contract: str): + """'Linux, Windows and macOS' may never appear unqualified.""" + lowered = contract.lower() + assert "ubuntu x64" in lowered + assert "windows x64" in lowered + assert "macos arm64" in lowered + for banned in ( + "works on linux, windows and macos", + "supported on linux, windows, and macos", + "all platforms", + "any platform", + ): + assert banned not in lowered, f"unqualified OS claim: {banned!r}" + + +def test_http_is_described_but_never_qualified(contract: str): + assert "unauthenticated" in contract.lower() + assert "no HTTP claim is derived from stdio evidence" in contract + for claim in re_mod.claim_rows(re_mod.load_claims()): + assert claim.get("transport") != "http", ( + "an HTTP claim needs live HTTP acceptance evidence, which the gate " + "does not produce" + ) + + +def test_no_upgrade_or_rollback_claim_is_made(contract: str): + assert "## 4. Upgrade qualification" in contract + assert "**None.**" in contract + assert "W14 has not run" in contract + + +def test_undetectability_is_refused(contract: str): + lowered = contract.lower() + assert "not claimed undetectable" in lowered or ( + "not promise universal undetectability" in lowered + ) + assert "f-774" in lowered + + +def test_the_limitations_register_names_every_required_area(contract: str): + """§2.5 enumerates what the register must contain; each id must appear.""" + required = ( + "E8-1", + "E8-2", + "E8-3", + "E8-4", + "E7-1", + "E7-6", + "F-181", + "F-165", + "close-path flake", + "F-773", + "F-774", + "F-776", + "native IME", + "HTTP transport", + "code-execution surface", + "missing interaction surface", + "live public web", + ) + for ident in required: + assert ident in contract, f"limitations register omits {ident!r}" + + +def test_the_register_names_w7s_exact_public_surface_exclusions(contract: str): + for exclusion in ( + "stale live handles", + "frame-targeted", + "redirect chains", + "truncation", + "downloads", + "SSE/WS", + "shadow-root", + ): + assert exclusion in contract, f"W7 exclusion missing: {exclusion!r}" + + +def test_every_unrun_workstream_is_declared(contract: str): + for workstream in ( + "W6", + "W7", + "W8", + "W9", + "W10", + "W11", + "W12", + "W13", + "W14", + "W15", + "W16", + ): + assert workstream in contract, f"{workstream} is not declared in the register" + + +def test_the_matrix_table_matches_the_ledgers_required_cells(contract: str): + for spec in gen.matrix_rows(): + assert f"| `{spec.job}` | `{spec.cell}` |" in contract, ( + f"required cell {spec.key} is missing from the contract matrix" + ) + + +def test_the_contract_names_the_version_it_ships(contract: str): + """This is a shipping document, not a draft: it names its own version.""" + version = gen.release_version() + assert f"# Release contract — version {version}" in contract + assert version == gen.release_version(), "the version must be read, not typed" + + +def test_the_breaking_change_is_prominent_not_buried(contract: str): + """The renamed knobs are the most user-visible thing in this release. + + The env var fails SILENTLY on upgrade — nothing errors, the configured cap + just stops applying — so it belongs above the tables, not in a register row. + """ + assert "### Breaking change from 1.x" in contract + assert "STEALTH_MCP_SESSION_STORAGE_CAP_GB" in contract + assert "STEALTH_MCP_BROWSER_SESSION_STORAGE_CAP_GB" in contract + assert "--session-cap-gb" in contract + assert "--browser-session-cap-gb" in contract + assert "no back-compatible alias" in contract + header_end = contract.index("## 1. The qualified matrix") + assert contract.index("### Breaking change from 1.x") < header_end, ( + "the breaking change must appear before the tables, not after them" + ) + + +def test_the_blind_push_property_is_explicitly_not_claimed(contract: str): + """§0.2 reserves 'green ⇒ blindly pushable' for evidence that does not exist.""" + assert "does not authorize a blind push" in contract + assert "None of the three is established here" in contract + + +def test_unrun_workstreams_read_as_not_evidenced_not_as_a_roadmap(contract: str): + assert "NOT EVIDENCED in this release" in contract + assert contract.count("NOT EVIDENCED in this release") >= 11, ( + "every workstream that produced no evidence must say so in its own row" + ) + for row in gen.LIMITATIONS: + if "NOT EVIDENCED" in row.area: + assert "may not infer" in row.evidence + + +def test_the_contract_says_it_is_generated(contract: str): + assert contract.startswith("", + "", + "", + f"# Release contract — version {release_version()}", + "", + "This is the contract for the version recorded in `pyproject.toml` at", + "this commit. A tagged run fails unless the tag equals that version,", + "so the two cannot disagree.", + "", + "What a green `release-gate` check does and does not authorize. Every", + "number below is derived: the served-tool count from the live", + "`SECTION_TOOLS` registry, the qualified count from the per-tool claim", + "ledger, and the matrix from the required cells of the", + "`release-evidence/v1` aggregate. Nothing here is hand-typed, and no", + "prose document — plan, finding, or README — can qualify anything.", + "", + "> **At the release SHA recorded in the ledger, this gate qualifies " + f"{qualified} of the {served} served MCP tools**, on the cells each", + "> row names.", + "", + "**Read that number carefully, in both directions.** It does *not* say", + f"{served - int(qualified or 0)} tools are untested. Every served tool", + "is driven against **real Chrome** by the E2E suite — a set-equality", + "tripwire keeps that coverage complete, and §5 shows each tool's actual", + "evidence. What those tools lack is a *per-tool* assertion over the", + "**stdio transport a user actually speaks**, because the suite drives", + "them through the in-process function seam instead. plan_RELEASE §2.5", + "does not let in-process evidence license a transport claim, so they", + "are `served-unqualified`: tested, but not proved at the wire. Closing", + "that is a question of where tests run, not of a product defect — see", + "F-776.", + "", + "### Breaking change from 1.x — read this before upgrading", + "", + "Two knobs were **renamed with no back-compatible alias**:", + "", + "| 1.x | now | effect if you keep using the old one |", + "|---|---|---|", + "| `STEALTH_MCP_SESSION_STORAGE_CAP_GB` " + "| `STEALTH_MCP_BROWSER_SESSION_STORAGE_CAP_GB` " + "| the variable is ignored — the cap silently returns to its default |", + "| `--session-cap-gb` | `--browser-session-cap-gb` " + "| the CLI rejects the unknown flag |", + "", + "The environment variable is the dangerous one: nothing errors, your", + "configured storage cap simply stops applying. Rename it before you", + "upgrade.", + "", + "### What this contract is NOT", + "", + "plan_RELEASE reserved a specific property — *a green gate is a", + "faithful stand-in for a manual pass, so you may push blind* — for a", + "workstream that has **not run**. That property rests on three things:", + "manual-QA parity, proven flake-freedom, and mutation-informed test", + "strength. **None of the three is established here**, and one required", + "cell has a known flake (below). This gate is strong on what it covers:", + "the real stdio wire path, three OSes, the exact published artifacts,", + "and offline stealth invariants. It is not a substitute for a human", + "release pass, and this document does not authorize a blind push.", + "", + ] + ) + + +def _matrix_section() -> str: + lines = [ + "## 1. The qualified matrix", + "", + "Exactly these GitHub-hosted cells, and nothing else. Each one writes a", + "`release-evidence/v1` child record; the `release-evidence` job refuses to", + "aggregate unless every cell below is present, current, and successful.", + "", + "| Job | Matrix cell | Runner | Python | Chrome | What the cell proves |", + "|---|---|---|---|---|---|", + ] + for spec in matrix_rows(): + if spec.expects_launched_chrome: + chrome = "launched + identity asserted" + elif spec.expects_chrome: + chrome = "resolved only (no launch)" + else: + chrome = "—" + # An unpinned cell records the interpreter it actually ran; pretending + # it pinned one would be a claim nothing checks. + python = spec.python_version or "image `python3` (recorded per run)" + lines.append( + f"| `{spec.job}` | `{spec.cell}` | {spec.label} | {python} " + f"| {chrome} | {_md_escape(spec.proves)} |" + ) + lines.extend( + [ + "", + "The exact Chrome Stable executable path and version are **not** written", + "into this document: they are image-provided and change under us. They", + "are recorded per run in", + "`release-evidence//integration/.json`, and the", + "integration cell asserts that production's own auto-discovery and the", + "CDP `Browser.getVersion` of the launched browser agree with them.", + "", + ] + ) + return "\n".join(lines) + + +def _exclusions_section() -> str: + return "\n".join( + [ + "## 2. What the matrix explicitly excludes", + "", + "These are not 'probably fine'. They are **untested**:", + "", + "- untested Linux distributions (the gate runs the hosted Ubuntu image", + " only) and any self-hosted runner;", + "- Windows ARM64 and Intel macOS;", + "- IPv6-only loopback;", + "- non-Stable Chrome channels (Beta, Dev, Canary, Chromium, Edge);", + "- future GitHub runner images — a qualification is a statement about", + " the image identity recorded in that run's ledger, not a standing", + " promise about `*-latest`.", + "", + "The phrase 'Linux, Windows and macOS' may only ever appear here with", + "those qualifiers attached: the qualified cells are **Ubuntu x64,", + "Windows x64 and macOS ARM64**, each as provided by the GitHub-hosted", + "image whose id the ledger records.", + "", + ] + ) + + +def _transport_section() -> str: + stdio_cells = [ + spec.label + for spec in matrix_rows() + if spec.job in release_evidence.TRANSPORT_JOBS + ] + return "\n".join( + [ + "## 3. Transport qualification", + "", + "### stdio (the transport a user actually gets)", + "", + "Qualified on: " + ", ".join(f"**{cell}**" for cell in stdio_cells) + ".", + "", + "The real-stdio lane spawns the installed console launcher, completes", + "the `initialize` handshake, lists tools, and drives one canonical", + "journey against real Chrome. macOS/ARM64 is **excluded** from this", + "lane under F-773 — excluded rather than xfailed, because an xfail", + "would let a green gate imply macOS coverage.", + "", + "That journey is a *representative* proof that the wire path works. It", + "is deliberately **not** per-tool evidence (plan_RELEASE §2.5), which is", + "why it qualifies the transport but no individual tool.", + "", + "### HTTP (described, not qualified)", + "", + "The server also supports `--transport http`. It is **unauthenticated**", + "by design and binds loopback by default. It is described here, not", + "qualified: no HTTP claim is derived from stdio evidence, and the gate", + "runs no live HTTP acceptance test. Anything able to reach that port", + "drives the browser with the caller's full privileges.", + "", + "## 4. Upgrade qualification", + "", + "**None.** W14 has not run: this contract makes no upgrade, migration,", + "or rollback claim from the literal N-1 stable tag or from any other", + "version. Installing over an existing installation is unqualified.", + "", + ] + ) + + +def _tool_section(counts: dict[str, object]) -> str: + rows = tool_rows() + lines = [ + "## 5. The served tool surface", + "", + f"- served by the registry: **{counts.get('served_total')}**", + f"- `release-qualified-success`: **{counts.get('release_qualified')}**", + f"- `served-unqualified`: **{counts.get('served_unqualified')}**", + f"- `not-served`: **{counts.get('not_served')}**", + "", + "A tool is `release-qualified-success` only when a row names the precise", + "user outcome, a fully-qualified passing node, the required transport, the", + "fixture or site shape, and the required OS cells — and the ledger shows", + "each of them as current-run success evidence. A schema or type assertion,", + "a `.fn`-only call, the representative journey, an error-only test, an", + "exemption, or a characterization **cannot** satisfy that bar.", + "", + "A qualified row is qualified **only on the cells its claim names**, and", + "the ledger enforces that: a `stdio` claim must be evidenced by the", + "transport lane, which runs on **Linux/X64 and Windows/X64 only**. macOS", + "ARM64 is excluded from that lane under F-773, so no per-tool stdio claim", + "in this document is qualified on three cells — every one of them is", + "qualified on exactly two.", + "", + "`served-unqualified` does not mean broken, and it does not mean untested.", + "It means: the server serves the tool, the suite exercises it against real", + "Chrome, and the gate at this SHA does not prove the user-visible outcome", + "*over the transport the user speaks*. The evidence column below states", + "which of those is true for each row — it is a description, never a claim.", + "A row reading **no execution evidence** would be materially weaker than", + "the rest; if none appears below, no served tool is in that state.", + "", + "Unless a row names a specific defect, its tracking id is **F-776** and its", + "impact is the shared one: no per-tool real-transport success assertion", + "exists at this SHA.", + "", + "| Tool | Section | State | Strongest current evidence | Tracking id |", + "|---|---|---|---|---|", + ] + lines.extend( + f"| `{row.tool}` | {row.section} | {row.state} | {_md_escape(row.tier)} " + f"| {_md_escape(row.tracking_id)} |" + for row in rows + ) + lines.extend(["", "### Tools whose row carries a specific defect", ""]) + lines.append("| Tool | Tracking id | User impact |") + lines.append("|---|---|---|") + for row in rows: + if row.tracking_id in ("F-776", "—"): + continue + lines.append( + f"| `{row.tool}` | {_md_escape(row.tracking_id)} " + f"| {_md_escape(row.impact)} |" + ) + lines.append("") + return "\n".join(lines) + + +def _limitations_section() -> str: + lines = [ + "## 6. Limitations register", + "", + "Everything a reader must know before trusting a green check. Rows are not", + "removed to shorten the document; a row leaves only when the thing it", + "describes actually closes.", + "", + "| Id | Area | Status | User impact | Evidence status |", + "|---|---|---|---|---|", + ] + lines.extend( + f"| {_md_escape(item.ident)} | {_md_escape(item.area)} " + f"| {_md_escape(item.status)} | {_md_escape(item.impact)} " + f"| {_md_escape(item.evidence)} |" + for item in LIMITATIONS + ) + lines.append("") + return "\n".join(lines) + + +def _ceiling_section() -> str: + return "\n".join( + [ + "## 7. The ceiling", + "", + "This contract does **not** promise that the server works on any site,", + "or that it will keep working. The open web and Chrome change", + "adversarially; a green gate is a statement about one SHA, one set of", + "recorded runner images, one recorded Chrome Stable build, and the", + "fixture shapes named above.", + "", + "It does **not** promise universal undetectability. The offline stealth", + "predicates passed their failing controls on all three cells — that is", + "sensitivity, not invisibility — and F-774 records a real residual", + "client-hint tell in the headless UA override.", + "", + "It does **not** claim the gate is flake-free — and on current", + "measurement it is not close. TWO distinct gate flakes are on", + "record. A Chrome cold-spawn race on the Linux runner has failed", + "first attempts in three different cells, including", + "`transport (Linux/X64)` — one of the two cells that carry the", + "qualified stdio claims. And F-779: `integration (macOS/ARM64)`", + "fails at teardown with `Event loop is closed` in **2 of 8**", + "consecutive runs, taking the `release-gate` aggregate down with it", + "each time. Its code independence is settled — one failure landed on", + "a commit containing nothing but a markdown file.", + "", + "Read that consequence carefully, because it is the honest headline", + "of this release: **the aggregate check is currently red about one", + "run in three for reasons that have nothing to do with the change", + "under review.** plan_RELEASE §0.2 makes flake-freedom one of the", + "three properties behind 'green ⇒ blindly pushable'. Until F-779 is", + "closed, a green check is evidence about that run only, and a red", + "check is not by itself evidence about your change. The workstream", + "that owns flake quarantine has not run.", + "", + "A green check is also evidence about ONE run attempt. The evidence", + "ledger binds every cell record to a single `run_id` + `run_attempt`,", + "so re-running only the failed jobs cannot produce a certificate —", + "the surviving records belong to the earlier attempt and are refused.", + "Re-run the whole workflow, or the gate stays red on identity, not on", + "the retried failure.", + "", + "Live public sites and detector scores are read-only informational", + "observations. They never license a deterministic claim, and no such", + "observation runs in this gate.", + "", + "If a claim you need is not written above, the honest answer is that", + "this gate does not make it.", + "", + ] + ) + + +def render_contract() -> str: + """The full contract text — the public API tests and W11 both call.""" + counts = release_evidence.tool_surface(release_evidence.load_claims()) + return "".join( + [ + _header(counts), + "\n", + _matrix_section(), + "\n", + _exclusions_section(), + "\n", + _transport_section(), + "\n", + _tool_section(counts), + "\n", + _limitations_section(), + "\n", + _ceiling_section(), + ] + ) + + +def check_contract() -> list[str]: + """Return the drift problems between the file on disk and a fresh render.""" + rendered = render_contract() + if not CONTRACT_PATH.is_file(): + return [f"{CONTRACT_PATH} does not exist"] + current = CONTRACT_PATH.read_text(encoding="utf-8") + if current == rendered: + return [] + return [ + f"{CONTRACT_PATH.name} is stale: regenerate with " + f"`uv run python tools/gen_release_contract.py --write`" + ] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--write", action="store_true", help="write the contract") + parser.add_argument("--check", action="store_true", help="fail on drift") + args = parser.parse_args(argv) + if args.write: + CONTRACT_PATH.write_text(render_contract(), encoding="utf-8", newline="\n") + print(f"wrote {CONTRACT_PATH}") + return 0 + if args.check: + problems = check_contract() + for problem in problems: + print(f"::error::gen_release_contract: {problem}") + return 1 if problems else 0 + print(render_contract(), end="") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/release_evidence.py b/tools/release_evidence.py new file mode 100644 index 0000000..040a32a --- /dev/null +++ b/tools/release_evidence.py @@ -0,0 +1,1232 @@ +#!/usr/bin/env python3 +"""The ONE parser/generator for the ``release-evidence/v1`` ledger (plan_RELEASE W5). + +Nothing else in this repository may parse, write, or interpret the ledger: the +release contract generator (:mod:`gen_release_contract`), W8's parity gate, and +W11's docs checker all import *this* module. A second reader would be a second +truth, and the whole point of the ledger is that there is exactly one. + +What the ledger is +------------------ +Every required job/matrix cell of ``.github/workflows/release-gate.yml`` writes +exactly one child record:: + + release-evidence///.json + +and the ``release-evidence`` job writes the conditional aggregate:: + + release-evidence//release-gate/aggregate.json + +The aggregate is a direct ``needs:`` edge of the ``release-gate`` check *in +addition to* every child job it validates, so the ledger can never turn a failed +job green — it can only ever turn a green run red. + +Fail-closed by construction +--------------------------- +:func:`build_aggregate` refuses to emit a successful aggregate when any of the +following holds (each has a negative test in ``tests/test_release_evidence.py``): + +* a required child record is missing, duplicated, or unreadable; +* a child record exists for a cell that is not declared in :data:`REQUIRED_CELLS`; +* a child's own ``job.id``/``job.matrix_cell`` disagree with its path; +* a child's ``release_sha`` is not the SHA being qualified (stale evidence); +* a child's ``workflow.run_id``/``run_attempt`` are not this run's; +* a child's ``job.terminal_outcome`` is anything other than ``success``; +* a browser cell recorded no Chrome identity, or a cell that must prove a + *launch* recorded no launched major version; +* a cell that must run pytest recorded no pytest block (or vice versa); +* a recorded artifact or JUnit hash does not match the bytes on disk; +* an ``MQ`` id is malformed, duplicated inside a record, or a declared required + MQ id is absent from the whole run; +* a tool claim cites a node that did not execute-and-pass on every cell it + claims, cites the representative journey, or claims a transport the citing + job cannot evidence. + +Deliberate schema notes +----------------------- +``chrome`` is ``null`` for a non-browser job (plan_RELEASE §2.5 says so +explicitly). ``pytest`` is ``null`` for a job that runs no pytest at all +(``quality``, ``known-gaps``, ``build-dist``, ``package-verify``, +``install-smoke`` — the last drives the journey through ``tools/install_smoke.py`` +rather than through pytest). Which cells may be null is *declared* per cell in +:data:`REQUIRED_CELLS`, so "null" is never a way to omit evidence a cell owes. + +``executed_node_ids`` holds every node the cell ran to a terminal result; +``skipped``/``xfail``/``failed`` are recorded separately and a claimed node that +appears in any of them is rejected. A skipped node is *not* in +``executed_node_ids`` — it was collected, not executed. + +Stdlib only: this runs on runners that have not installed the project. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import re +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path +from xml.etree import ElementTree + +SCHEMA = "release-evidence/v1" +CLAIMS_SCHEMA = "release-evidence/v1#tool-claims" +AGGREGATE_JOB = "release-gate" +AGGREGATE_CELL = "aggregate" + +TOOLS_DIR = Path(__file__).resolve().parent +REPO_ROOT = TOOLS_DIR.parent +CLAIMS_PATH = TOOLS_DIR / "release_tool_claims.json" + +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +HASH_RE = re.compile(r"^[0-9a-f]{64}$") +NUMERIC_RE = re.compile(r"^[0-9]+$") +CELL_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +PYTHON_RE = re.compile(r"^3\.\d+(\.\d+)?$") +VERSION_RE = re.compile(r"^\d+(\.\d+)*$") +NODE_RE = re.compile(r"^[^:]+\.py::.+$") +MQ_RE = re.compile(r"^MQ-\d+$") +# Real GitHub-hosted image ids carry hyphens and dots (`ubuntu24`, `macos15`, +# `win25-vs2026`). The check that matters is that there IS one: a self-hosted +# runner exposes no ImageOS at all, and an empty value still fails. +IMAGE_OS_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") + +TERMINAL_OUTCOMES = frozenset({"success", "failure", "cancelled", "skipped"}) +WORKFLOW_EVENTS = frozenset( + {"pull_request", "push", "workflow_dispatch", "schedule", "merge_group", "release"} +) +ARTIFACT_KINDS = frozenset( + { + "runner-identity", + "chrome-identity", + "coverage", + "junit", + "build-manifest", + "package-verify", + "install-smoke", + } +) +TOOL_STATES = ("release-qualified-success", "served-unqualified", "not-served") +TRANSPORTS = frozenset({"stdio", "http"}) +CLAIMS_KEYS = frozenset( + { + "schema", + "note", + "qualified", + "default_note", + "served_unqualified_notes", + "not_served", + } +) + +RECORD_KEYS = frozenset( + { + "schema", + "release_sha", + "workflow", + "job", + "runner", + "python_version", + "chrome", + "pytest", + "artifacts", + "mq_ids", + } +) + +# W8 populates this as MQ steps acquire runtime evidence. It is a real check +# today: a declared id that no child records fails the aggregate. +REQUIRED_MQ_IDS: frozenset[str] = frozenset() + +# plan_RELEASE §2.1/§2.5: the representative journey is ONE node covering many +# tools. It is release evidence that the wire path works; it is explicitly NOT +# per-tool evidence, so no tool row may cite it. +NON_PER_TOOL_NODES = frozenset( + {"tests/test_e2e_transport.py::test_real_stdio_release_gate_journey"} +) + + +@dataclass(frozen=True) +class CellSpec: + """One required job/matrix cell of the release gate. + + ``expects_pytest``/``expects_chrome``/``expects_launched_chrome`` are the + fail-closed declarations: a cell that owes pytest evidence cannot satisfy the + aggregate with ``pytest: null``, and a cell that owes a *launched* browser + cannot satisfy it by merely resolving the executable on disk. + """ + + job: str + cell: str + runner_os: str + runner_arch: str + python_version: str + expects_pytest: bool + expects_chrome: bool + expects_launched_chrome: bool + proves: str + + @property + def key(self) -> str: + return f"{self.job}/{self.cell}" + + @property + def label(self) -> str: + return f"{self.runner_os}/{self.runner_arch}" + + +# (cell name, runner.os, runner.arch) for the three qualified runners. +LINUX = ("Linux-X64", "Linux", "X64") +WINDOWS = ("Windows-X64", "Windows", "X64") +MACOS = ("macOS-ARM64", "macOS", "ARM64") +ALL_OS = (LINUX, WINDOWS, MACOS) +# The ubuntu-only jobs run whatever ``python3`` the image provides; they pin no +# interpreter, so their record carries the version that actually ran and the +# validator checks its SHAPE rather than an invented equality. +UNPINNED = "" + + +def _spec( + job: str, + cell: tuple[str, str, str], + python: str, + flags: str, + proves: str, +) -> CellSpec: + """One cell. ``flags``: ``p`` owes pytest, ``c`` Chrome, ``l`` a LAUNCH.""" + return CellSpec( + job, + cell[0], + cell[1], + cell[2], + python, + expects_pytest="p" in flags, + expects_chrome="c" in flags, + expects_launched_chrome="l" in flags, + proves=proves, + ) + + +def _build_required_cells() -> tuple[CellSpec, ...]: + """The exact required job/cell key set the aggregate demands. + + Mirrors ``release-gate.yml``; ``tests/test_release_workflows.py`` asserts the + two never drift apart, so adding a matrix cell without adding its evidence + edge is a red test rather than a silent hole. + """ + one = ("default", "Linux", "X64") + specs: list[CellSpec] = [ + _spec("quality", one, UNPINNED, "", "lint/type/vulture/owner/budget gates"), + _spec("known-gaps", one, UNPINNED, "", "the declared gaps, in the check list"), + _spec("build-dist", one, UNPINNED, "", "the ONE build + its hashed manifest"), + _spec( + "package-verify", + one, + UNPINNED, + "", + "downloaded-bytes re-check + three bite proofs", + ), + ] + specs += [ + _spec( + "unit-tests", + (f"{cell[0]}-py{py}", cell[1], cell[2]), + py, + "p", + "hermetic unit suite (`-m 'not integration'`)", + ) + for py in ("3.11", "3.12", "3.13") + for cell in ALL_OS + ] + specs += [ + _spec( + "coverage", + cell, + "3.12", + "p", + "per-OS coverage floor (no merged report hides a red OS)", + ) + for cell in ALL_OS + ] + specs += [ + _spec( + "integration", + cell, + "3.12", + "pcl", + "real-Chrome integration suite + Chrome identity" + + (" MINUS the transport journey (F-773)" if cell is MACOS else ""), + ) + for cell in ALL_OS + ] + specs += [ + _spec( + "offline-stealth", + cell, + "3.12", + "pcl", + "offline stealth predicates and their failing controls", + ) + for cell in ALL_OS + ] + specs += [ + _spec( + "transport", + cell, + "3.12", + "pcl", + "real-stdio JSON-RPC journey against real Chrome", + ) + for cell in (LINUX, WINDOWS) + ] + specs += [ + _spec( + "install-smoke", + (f"{kind}-{cell[0]}", cell[1], cell[2]), + "3.12", + "c" if cell is MACOS else "cl", + f"clean install of the exact {kind} + " + + ("handshake only (NO navigation, F-773)" if cell is MACOS else "journey"), + ) + for kind in ("wheel", "sdist") + for cell in ALL_OS + ] + return tuple(specs) + + +REQUIRED_CELLS: tuple[CellSpec, ...] = _build_required_cells() +CELLS_BY_KEY: dict[str, CellSpec] = {spec.key: spec for spec in REQUIRED_CELLS} +REQUIRED_KEYS: tuple[str, ...] = tuple(sorted(CELLS_BY_KEY)) + +# A stdio claim is only believable from a job whose selector runs the real-stdio +# lane. Transport-ness is proved by WHICH JOB executed the node, never by a +# hand-written label on the claim. +TRANSPORT_JOBS = frozenset({"transport"}) + + +# ── hashing / small helpers ───────────────────────────────────────────────── +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def dumps(payload: object) -> str: + """Deterministic JSON: sorted keys, fixed indent, trailing newline.""" + return json.dumps(payload, indent=2, sort_keys=True) + "\n" + + +def _sorted_unique(values: list[str]) -> list[str]: + return sorted(set(values)) + + +def _as_dict(value: object) -> dict[str, object] | None: + return value if isinstance(value, dict) else None + + +def _str_field(block: dict[str, object], name: str) -> str: + value = block.get(name) + return value if isinstance(value, str) else "" + + +# ── JUnit parsing ─────────────────────────────────────────────────────────── +def _node_id(testcase: ElementTree.Element) -> str: + """Rebuild pytest's fully-qualified node id from an xunit2 ````.""" + file_attr = (testcase.get("file") or "").replace("\\", "/") + name = testcase.get("name") or "" + classname = testcase.get("classname") or "" + if not file_attr: + # No file attribute (junit_family=xunit1): fall back to the dotted name. + return f"{classname}::{name}" if classname else name + module_dotted = ( + file_attr[:-3].replace("/", ".") if file_attr.endswith(".py") else "" + ) + if module_dotted and classname.startswith(f"{module_dotted}."): + inner = classname[len(module_dotted) + 1 :].replace(".", "::") + return f"{file_attr}::{inner}::{name}" + return f"{file_attr}::{name}" + + +def parse_junit(path: Path) -> dict[str, list[str]]: + """Split a JUnit XML report into executed / skipped / xfail / failed nodes. + + ``executed`` holds every node that ran to a terminal result. A node that was + collected but never ran (``skipped``) is deliberately absent from it. + """ + tree = ElementTree.parse(path) # noqa: S314 PERMANENT(the file is this run's own pytest output; defusedxml is not a permitted new dependency) + executed: list[str] = [] + skipped: list[str] = [] + xfail: list[str] = [] + failed: list[str] = [] + for testcase in tree.iter("testcase"): + node = _node_id(testcase) + skip_el = testcase.find("skipped") + if skip_el is not None: + if (skip_el.get("type") or "") == "pytest.xfail": + xfail.append(node) + executed.append(node) + else: + skipped.append(node) + continue + if testcase.find("failure") is not None or testcase.find("error") is not None: + failed.append(node) + executed.append(node) + return { + "executed_node_ids": _sorted_unique(executed), + "skipped": _sorted_unique(skipped), + "xfail": _sorted_unique(xfail), + "failed": _sorted_unique(failed), + } + + +# ── record construction ───────────────────────────────────────────────────── +@dataclass(frozen=True) +class EmitSpec: + """Everything one cell needs to write its child record.""" + + out_root: Path + release_sha: str + workflow_name: str + run_id: str + run_attempt: str + event: str + job_id: str + matrix_cell: str + terminal_outcome: str + runner_identity: Path | None + chrome_identity: Path | None + chrome_launched: bool + junit: Path | None + artifacts: tuple[tuple[str, Path], ...] + mq_ids: tuple[str, ...] + + +def _runner_block(spec: EmitSpec) -> dict[str, object]: + if spec.runner_identity is None: + return {"os": "", "arch": "", "image_os": "", "image_version": ""} + data = json.loads(spec.runner_identity.read_text(encoding="utf-8")) + return { + "os": data.get("runner_os", ""), + "arch": data.get("runner_arch", ""), + "image_os": data.get("image_os", ""), + "image_version": data.get("image_version", ""), + } + + +def _python_version(spec: EmitSpec) -> str: + if spec.runner_identity is None: + return platform.python_version() + data = json.loads(spec.runner_identity.read_text(encoding="utf-8")) + value = data.get("python_version", "") + return value if isinstance(value, str) else "" + + +def _chrome_block(spec: EmitSpec) -> dict[str, object] | None: + if spec.chrome_identity is None: + return None + data = json.loads(spec.chrome_identity.read_text(encoding="utf-8")) + version = data.get("version", "") + version = version if isinstance(version, str) else "" + launched: int | None = None + if spec.chrome_launched and version: + head = version.split(".", 1)[0] + launched = int(head) if head.isdigit() else None + return { + "path": data.get("path", ""), + "executable_version": version, + "launched_major": launched, + } + + +def _copy_artifacts(spec: EmitSpec) -> list[dict[str, object]]: + """Copy each cited artifact next to the record and hash the copy. + + The record's ``path`` is relative to the evidence root, so the aggregate can + re-hash exactly the bytes this cell recorded rather than trusting a path that + only existed inside one job's workspace. + """ + dest_dir = spec.out_root / spec.release_sha / spec.job_id / "artifacts" + dest_dir = dest_dir / spec.matrix_cell + entries: list[dict[str, object]] = [] + for kind, source in spec.artifacts: + if not source.is_file(): + print(f"::warning::release_evidence: artifact {source} is absent") + continue + dest_dir.mkdir(parents=True, exist_ok=True) + target = dest_dir / source.name + shutil.copyfile(source, target) + rel = target.relative_to(spec.out_root / spec.release_sha).as_posix() + entries.append( + { + "name": source.name, + "path": rel, + "kind": kind, + "sha256": sha256_file(target), + } + ) + return sorted(entries, key=lambda item: str(item["name"])) + + +def build_record(spec: EmitSpec) -> dict[str, object]: + pytest_block: dict[str, object] | None = None + if spec.junit is not None: + parsed = parse_junit(spec.junit) + pytest_block = {"junit_sha256": sha256_file(spec.junit), **parsed} + return { + "schema": SCHEMA, + "release_sha": spec.release_sha, + "workflow": { + "name": spec.workflow_name, + "run_id": spec.run_id, + "run_attempt": spec.run_attempt, + "event": spec.event, + }, + "job": { + "id": spec.job_id, + "matrix_cell": spec.matrix_cell, + "terminal_outcome": spec.terminal_outcome, + }, + "runner": _runner_block(spec), + "python_version": _python_version(spec), + "chrome": _chrome_block(spec), + "pytest": pytest_block, + "artifacts": _copy_artifacts(spec), + "mq_ids": sorted(spec.mq_ids), + } + + +# ── record validation ─────────────────────────────────────────────────────── +def _validate_workflow(value: object) -> list[str]: + block = _as_dict(value) + if block is None: + return ["workflow: not an object"] + problems: list[str] = [] + if set(block) != {"name", "run_id", "run_attempt", "event"}: + problems.append(f"workflow: unexpected field set {sorted(block)}") + if not _str_field(block, "name"): + problems.append("workflow.name: empty") + problems.extend( + f"workflow.{field}: {block.get(field)!r} is not numeric" + for field in ("run_id", "run_attempt") + if not NUMERIC_RE.match(_str_field(block, field)) + ) + if _str_field(block, "event") not in WORKFLOW_EVENTS: + problems.append(f"workflow.event: {block.get('event')!r} is not a known event") + return problems + + +def _validate_job(value: object, *, expect_key: str) -> list[str]: + block = _as_dict(value) + if block is None: + return ["job: not an object"] + problems: list[str] = [] + if set(block) != {"id", "matrix_cell", "terminal_outcome"}: + problems.append(f"job: unexpected field set {sorted(block)}") + job_id = _str_field(block, "id") + cell = _str_field(block, "matrix_cell") + if not CELL_RE.match(cell): + problems.append(f"job.matrix_cell: {cell!r} is malformed") + key = f"{job_id}/{cell}" + if key != expect_key: + problems.append(f"job: record says {key!r} but its path says {expect_key!r}") + if _str_field(block, "terminal_outcome") not in TERMINAL_OUTCOMES: + problems.append( + f"job.terminal_outcome: {block.get('terminal_outcome')!r} is not a " + f"known outcome" + ) + return problems + + +def _validate_runner(value: object, spec: CellSpec) -> list[str]: + block = _as_dict(value) + if block is None: + return ["runner: not an object"] + problems: list[str] = [] + if set(block) != {"os", "arch", "image_os", "image_version"}: + problems.append(f"runner: unexpected field set {sorted(block)}") + if _str_field(block, "os") != spec.runner_os: + problems.append( + f"runner.os: {block.get('os')!r} != declared cell {spec.runner_os!r}" + ) + if _str_field(block, "arch") != spec.runner_arch: + problems.append( + f"runner.arch: {block.get('arch')!r} != declared cell {spec.runner_arch!r}" + ) + if not IMAGE_OS_RE.match(_str_field(block, "image_os")): + problems.append( + f"runner.image_os: {block.get('image_os')!r} is not a GitHub-hosted " + f"image id (a runner without one is outside the qualified matrix)" + ) + if not _str_field(block, "image_version"): + problems.append("runner.image_version: empty") + return problems + + +def _validate_chrome(value: object, spec: CellSpec) -> list[str]: + if value is None: + if spec.expects_chrome: + return [f"chrome: null on browser cell {spec.key!r}"] + return [] + block = _as_dict(value) + if block is None: + return ["chrome: not an object"] + problems: list[str] = [] + if not spec.expects_chrome: + problems.append(f"chrome: recorded on non-browser cell {spec.key!r}") + if set(block) != {"path", "executable_version", "launched_major"}: + problems.append(f"chrome: unexpected field set {sorted(block)}") + if not _str_field(block, "path"): + problems.append("chrome.path: empty") + if not VERSION_RE.match(_str_field(block, "executable_version")): + problems.append( + f"chrome.executable_version: {block.get('executable_version')!r} is " + f"not a version" + ) + launched = block.get("launched_major") + if spec.expects_launched_chrome and not isinstance(launched, int): + problems.append( + f"chrome.launched_major: cell {spec.key!r} must prove a LAUNCHED " + f"browser, not merely a resolved executable" + ) + if not spec.expects_launched_chrome and launched is not None: + problems.append( + f"chrome.launched_major: cell {spec.key!r} is declared " + f"non-launching but recorded a launch" + ) + return problems + + +def _validate_node_list(block: dict[str, object], field: str) -> list[str]: + value = block.get(field) + if not isinstance(value, list): + return [f"pytest.{field}: not a list"] + problems: list[str] = [] + nodes = [item for item in value if isinstance(item, str)] + if len(nodes) != len(value): + problems.append(f"pytest.{field}: contains a non-string entry") + if nodes != sorted(nodes): + problems.append(f"pytest.{field}: not deterministically sorted") + if len(set(nodes)) != len(nodes): + problems.append(f"pytest.{field}: contains duplicates") + problems.extend( + f"pytest.{field}: {node!r} is not a fully-qualified node id" + for node in nodes + if not NODE_RE.match(node) + ) + return problems + + +def _validate_pytest(value: object, spec: CellSpec) -> list[str]: + if value is None: + if spec.expects_pytest: + return [f"pytest: null on cell {spec.key!r}, which owes pytest evidence"] + return [] + block = _as_dict(value) + if block is None: + return ["pytest: not an object"] + problems: list[str] = [] + if not spec.expects_pytest: + problems.append(f"pytest: recorded on non-pytest cell {spec.key!r}") + expected = {"junit_sha256", "executed_node_ids", "skipped", "xfail", "failed"} + if set(block) != expected: + problems.append(f"pytest: unexpected field set {sorted(block)}") + return problems + if not HASH_RE.match(_str_field(block, "junit_sha256")): + problems.append("pytest.junit_sha256: not a sha256 digest") + for field in ("executed_node_ids", "skipped", "xfail", "failed"): + problems.extend(_validate_node_list(block, field)) + return problems + + +def _validate_artifacts(value: object) -> list[str]: + if not isinstance(value, list): + return ["artifacts: not a list"] + problems: list[str] = [] + names: list[str] = [] + for index, item in enumerate(value): + entry = _as_dict(item) + if entry is None: + problems.append(f"artifacts[{index}]: not an object") + continue + if set(entry) != {"name", "path", "kind", "sha256"}: + problems.append(f"artifacts[{index}]: unexpected field set {sorted(entry)}") + continue + names.append(_str_field(entry, "name")) + if _str_field(entry, "kind") not in ARTIFACT_KINDS: + problems.append(f"artifacts[{index}].kind: {entry.get('kind')!r} unknown") + if not HASH_RE.match(_str_field(entry, "sha256")): + problems.append(f"artifacts[{index}].sha256: not a sha256 digest") + path = _str_field(entry, "path") + if not path or path.startswith("/") or ".." in path.split("/"): + problems.append(f"artifacts[{index}].path: {path!r} is not ledger-relative") + if names != sorted(names): + problems.append("artifacts: not deterministically sorted by name") + if len(set(names)) != len(names): + problems.append("artifacts: duplicate artifact name") + return problems + + +def _validate_mq_ids(value: object) -> list[str]: + if not isinstance(value, list): + return ["mq_ids: not a list"] + ids = [item for item in value if isinstance(item, str)] + problems: list[str] = [] + if len(ids) != len(value): + problems.append("mq_ids: contains a non-string entry") + if len(set(ids)) != len(ids): + problems.append("mq_ids: duplicate id inside one record") + if ids != sorted(ids): + problems.append("mq_ids: not deterministically sorted") + problems.extend(f"mq_ids: {mq!r} is malformed" for mq in ids if not MQ_RE.match(mq)) + return problems + + +def validate_record(record: object, *, expect_key: str) -> list[str]: + """Return every schema violation in one child record (empty == valid).""" + block = _as_dict(record) + if block is None: + return ["record: not a JSON object"] + spec = CELLS_BY_KEY.get(expect_key) + if spec is None: + return [f"record: {expect_key!r} is not a declared required cell"] + problems: list[str] = [] + missing = RECORD_KEYS - set(block) + unknown = set(block) - RECORD_KEYS + if missing: + problems.append(f"record: missing field(s) {sorted(missing)}") + if unknown: + problems.append(f"record: unknown field(s) {sorted(unknown)}") + if block.get("schema") != SCHEMA: + problems.append(f"schema: {block.get('schema')!r} != {SCHEMA!r}") + if not SHA_RE.match(_str_field(block, "release_sha")): + problems.append(f"release_sha: {block.get('release_sha')!r} is malformed") + if not PYTHON_RE.match(_str_field(block, "python_version")): + problems.append(f"python_version: {block.get('python_version')!r} is malformed") + if spec.python_version and _str_field(block, "python_version") != ( + spec.python_version + ): + problems.append( + f"python_version: {block.get('python_version')!r} != declared cell " + f"{spec.python_version!r}" + ) + problems.extend(_validate_workflow(block.get("workflow"))) + problems.extend(_validate_job(block.get("job"), expect_key=expect_key)) + problems.extend(_validate_runner(block.get("runner"), spec)) + problems.extend(_validate_chrome(block.get("chrome"), spec)) + problems.extend(_validate_pytest(block.get("pytest"), spec)) + problems.extend(_validate_artifacts(block.get("artifacts"))) + problems.extend(_validate_mq_ids(block.get("mq_ids"))) + return problems + + +# ── aggregate ─────────────────────────────────────────────────────────────── +@dataclass(frozen=True) +class AggregateSpec: + """The identity the aggregate demands every child agree with.""" + + root: Path + release_sha: str + workflow_name: str + run_id: str + run_attempt: str + event: str + claims_path: Path + + +def _discover_children(sha_dir: Path) -> tuple[dict[str, Path], list[str]]: + """Map ``job/cell`` → record path; report extras and duplicates.""" + found: dict[str, Path] = {} + problems: list[str] = [] + # Keys are path-derived, so an exact duplicate cannot exist on disk — but a + # case-variant CAN on a case-sensitive filesystem, and merging artifacts + # from many jobs is exactly where one would appear. Collision is checked + # case-insensitively BEFORE the declared-cell check so a second record for a + # cell reads as the duplicate it is rather than as an unrelated stray. + by_lowered: dict[str, str] = {} + for path in sorted(sha_dir.rglob("*.json")): + rel = path.relative_to(sha_dir).as_posix() + if rel.startswith(f"{AGGREGATE_JOB}/") or "/artifacts/" in rel: + continue + key = rel[: -len(".json")] + lowered = key.lower() + if lowered in by_lowered: + problems.append( + f"duplicate child record for {key!r} (already have " + f"{by_lowered[lowered]!r})" + ) + continue + by_lowered[lowered] = key + if key not in CELLS_BY_KEY: + problems.append(f"extra child record for undeclared cell {key!r}") + continue + found[key] = path + problems.extend( + f"missing child record for required cell {key!r}" + for key in REQUIRED_KEYS + if key not in found + ) + return found, problems + + +def _check_identity(record: dict[str, object], spec: AggregateSpec) -> list[str]: + problems: list[str] = [] + if _str_field(record, "release_sha") != spec.release_sha: + problems.append( + f"stale evidence: release_sha {record.get('release_sha')!r} != " + f"{spec.release_sha!r}" + ) + workflow = _as_dict(record.get("workflow")) or {} + if _str_field(workflow, "run_id") != spec.run_id: + problems.append( + f"foreign evidence: workflow.run_id {workflow.get('run_id')!r} != " + f"{spec.run_id!r}" + ) + if _str_field(workflow, "run_attempt") != spec.run_attempt: + problems.append( + f"foreign evidence: workflow.run_attempt " + f"{workflow.get('run_attempt')!r} != {spec.run_attempt!r}" + ) + job = _as_dict(record.get("job")) or {} + if _str_field(job, "terminal_outcome") != "success": + problems.append(f"non-success terminal outcome {job.get('terminal_outcome')!r}") + return problems + + +def _check_hashes(record: dict[str, object], sha_dir: Path) -> list[str]: + problems: list[str] = [] + artifacts = record.get("artifacts") + junit_recorded = "" + pytest_block = _as_dict(record.get("pytest")) + if pytest_block is not None: + junit_recorded = _str_field(pytest_block, "junit_sha256") + if not isinstance(artifacts, list): + return ["artifacts: not a list"] + junit_seen = False + for item in artifacts: + entry = _as_dict(item) + if entry is None: + continue + path = sha_dir / _str_field(entry, "path") + if not path.is_file(): + problems.append(f"artifact {entry.get('path')!r} was not uploaded") + continue + actual = sha256_file(path) + if actual != _str_field(entry, "sha256"): + problems.append( + f"artifact hash mismatch for {entry.get('path')!r}: recorded " + f"{entry.get('sha256')!r}, on disk {actual!r}" + ) + if _str_field(entry, "kind") == "junit": + junit_seen = True + if junit_recorded and actual != junit_recorded: + problems.append( + f"JUnit hash mismatch: pytest.junit_sha256 {junit_recorded!r} " + f"!= uploaded report {actual!r}" + ) + if junit_recorded and not junit_seen: + problems.append("pytest.junit_sha256 recorded but no junit artifact uploaded") + return problems + + +def _executed_index(children: dict[str, dict[str, object]]) -> dict[str, set[str]]: + """cell key → the node ids that executed AND passed on that cell.""" + index: dict[str, set[str]] = {} + for key, record in children.items(): + block = _as_dict(record.get("pytest")) + if block is None: + index[key] = set() + continue + bad: set[str] = set() + for field in ("skipped", "xfail", "failed"): + value = block.get(field) + if isinstance(value, list): + bad |= {item for item in value if isinstance(item, str)} + executed = block.get("executed_node_ids") + ran = ( + {item for item in executed if isinstance(item, str)} + if isinstance(executed, list) + else set() + ) + index[key] = ran - bad + return index + + +def _check_mq_ids(children: dict[str, dict[str, object]]) -> list[str]: + seen: set[str] = set() + for record in children.values(): + value = record.get("mq_ids") + if isinstance(value, list): + seen |= {item for item in value if isinstance(item, str)} + return [ + f"required MQ id {mq!r} has no runtime evidence in this run" + for mq in sorted(REQUIRED_MQ_IDS - seen) + ] + + +def verify_claims(claims: dict[str, object], index: dict[str, set[str]]) -> list[str]: + """Check every ``release-qualified-success`` claim against the real ledger. + + This is what stops the contract from claiming a tool the run did not prove: + the claim's node must have executed AND passed on every cell it names, must + not be the representative journey, and a ``stdio`` claim must be evidenced by + a job that actually drives the real-stdio lane. + """ + problems: list[str] = [] + for claim in claim_rows(claims): + node = str(claim.get("node_id", "")) + tool = str(claim.get("tool", "")) + cells = claim.get("required_cells") + if node in NON_PER_TOOL_NODES: + problems.append( + f"claim for {tool!r} cites the representative journey {node!r}, " + f"which plan_RELEASE §2.5 forbids as per-tool evidence" + ) + if not isinstance(cells, list) or not cells: + problems.append(f"claim for {tool!r} names no required cells") + continue + for cell in cells: + key = str(cell) + if key not in CELLS_BY_KEY: + problems.append(f"claim for {tool!r} names undeclared cell {key!r}") + continue + if str(claim.get("transport", "")) == "stdio" and ( + CELLS_BY_KEY[key].job not in TRANSPORT_JOBS + ): + problems.append( + f"claim for {tool!r} asserts stdio but cites {key!r}, which " + f"does not run the real-stdio lane" + ) + if node not in index.get(key, set()): + problems.append( + f"claim for {tool!r} cites {node!r} on {key!r}, where it did " + f"not execute and pass" + ) + return problems + + +def build_aggregate(spec: AggregateSpec) -> tuple[dict[str, object], list[str]]: + """Build the aggregate record and every reason it must not be trusted.""" + sha_dir = spec.root / spec.release_sha + problems: list[str] = [] + if not sha_dir.is_dir(): + return {}, [f"no evidence directory for release SHA {spec.release_sha!r}"] + paths, problems_found = _discover_children(sha_dir) + problems.extend(problems_found) + records: dict[str, dict[str, object]] = {} + children: list[dict[str, object]] = [] + for key in sorted(paths): + path = paths[key] + try: + record = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + problems.append(f"child {key!r} is unreadable: {exc}") + continue + problems.extend(f"{key}: {p}" for p in validate_record(record, expect_key=key)) + block = _as_dict(record) + if block is None: + continue + records[key] = block + problems.extend(f"{key}: {p}" for p in _check_identity(block, spec)) + problems.extend(f"{key}: {p}" for p in _check_hashes(block, sha_dir)) + children.append( + { + "key": key, + "path": path.relative_to(sha_dir).as_posix(), + "sha256": sha256_file(path), + } + ) + problems.extend(_check_mq_ids(records)) + claims = load_claims(spec.claims_path) + problems.extend(verify_claims(claims, _executed_index(records))) + surface = tool_surface(claims) + aggregate: dict[str, object] = { + "schema": SCHEMA, + "release_sha": spec.release_sha, + "workflow": { + "name": spec.workflow_name, + "run_id": spec.run_id, + "run_attempt": spec.run_attempt, + "event": spec.event, + }, + "job": { + "id": AGGREGATE_JOB, + "matrix_cell": AGGREGATE_CELL, + "terminal_outcome": "success" if not problems else "failure", + }, + "required_cells": list(REQUIRED_KEYS), + "children": sorted(children, key=lambda item: str(item["key"])), + "tool_surface": surface, + "problems": problems, + } + return aggregate, problems + + +# ── tool claim ledger ─────────────────────────────────────────────────────── +def registry_sections() -> dict[str, tuple[str, ...]]: + """section → served tool names, DERIVED from the live registry (never typed). + + ``SECTION_TOOLS`` is filled by ``@section_tool`` at registration time, so the + server module must be imported before it is read — exactly what + ``tests/test_doc_claims.py`` does. Importing it as a module (not via runpy) + registers each tool once. + """ + from stealth_chrome_devtools_mcp.embedded import server as _server + from stealth_chrome_devtools_mcp.embedded.tool_registry import SECTION_TOOLS + + assert _server is not None # noqa: S101 PERMANENT(the import is the point: it populates SECTION_TOOLS) + return {section: tuple(sorted(names)) for section, names in SECTION_TOOLS.items()} + + +def registry_tool_names() -> tuple[str, ...]: + """The served tool surface, DERIVED from the live registry (never typed).""" + return tuple(sorted({n for names in registry_sections().values() for n in names})) + + +def load_claims(path: Path | None = None) -> dict[str, object]: + target = path if path is not None else CLAIMS_PATH + data = json.loads(target.read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("schema") != CLAIMS_SCHEMA: + raise ValueError(f"{target}: not a {CLAIMS_SCHEMA} document") + return data + + +def claim_rows(claims: dict[str, object]) -> list[dict[str, object]]: + """The declared `release-qualified-success` rows (verified elsewhere).""" + rows = claims.get("qualified") + if not isinstance(rows, list): + return [] + return [row for row in rows if isinstance(row, dict)] + + +def _notes(claims: dict[str, object]) -> dict[str, object]: + notes = claims.get("served_unqualified_notes") + return notes if isinstance(notes, dict) else {} + + +def validate_claims_document( + claims: dict[str, object], names: tuple[str, ...] +) -> list[str]: + """Every served tool has exactly one state; no claim names an unknown tool.""" + problems: list[str] = [] + served = set(names) + unknown_keys = set(claims) - CLAIMS_KEYS + if unknown_keys: + problems.append(f"claims: unknown top-level field(s) {sorted(unknown_keys)}") + missing_keys = CLAIMS_KEYS - set(claims) + if missing_keys: + problems.append(f"claims: missing top-level field(s) {sorted(missing_keys)}") + seen: list[str] = [] + for row in claim_rows(claims): + tool = str(row.get("tool", "")) + seen.append(tool) + if tool not in served: + problems.append(f"qualified claim for unknown tool {tool!r}") + missing = { + "tool", + "outcome", + "transport", + "node_id", + "site_shape", + "required_cells", + } - set(row) + if missing: + problems.append(f"claim for {tool!r} is missing {sorted(missing)}") + if str(row.get("transport", "")) not in TRANSPORTS: + problems.append(f"claim for {tool!r} names an unknown transport") + if len(set(seen)) != len(seen): + problems.append("duplicate qualified claim for the same tool") + problems.extend( + f"served-unqualified note for unknown tool {tool!r}" + for tool in _notes(claims) + if tool not in served + ) + not_served = claims.get("not_served") + if isinstance(not_served, list): + problems.extend( + f"not-served entry {name!r} is actually served by the registry" + for name in not_served + if str(name) in served + ) + return problems + + +def tool_surface(claims: dict[str, object]) -> dict[str, object]: + """Counts for the contract headline — derived, never typed.""" + names = registry_tool_names() + qualified = {str(row.get("tool", "")) for row in claim_rows(claims)} + not_served = claims.get("not_served") + not_served_count = len(not_served) if isinstance(not_served, list) else 0 + return { + "served_total": len(names), + "release_qualified": len(qualified & set(names)), + "served_unqualified": len(set(names) - qualified), + "not_served": not_served_count, + } + + +# ── CLI ───────────────────────────────────────────────────────────────────── +def _artifact_pairs(values: list[str]) -> tuple[tuple[str, Path], ...]: + pairs: list[tuple[str, Path]] = [] + for value in values: + kind, _, raw = value.partition("=") + if kind not in ARTIFACT_KINDS or not raw: + raise SystemExit(f"--artifact must be =, got {value!r}") + pairs.append((kind, Path(raw))) + return tuple(pairs) + + +def _optional(path: str) -> Path | None: + """A cited input that a FAILED step never produced is reported, not raised. + + Emit runs with ``if: always()`` so a red cell still writes its record. When + an input is absent the record simply lacks that evidence, and + :func:`validate_record` fails the cell for the omission — the failure stays a + ledger problem instead of an emit traceback that hides it. + """ + if not path: + return None + candidate = Path(path) + if not candidate.is_file(): + print(f"::warning::release_evidence: {path} was not produced by this cell") + return None + return candidate + + +def _cmd_emit(args: argparse.Namespace) -> int: + key = f"{args.job_id}/{args.matrix_cell}" + if key not in CELLS_BY_KEY: + print(f"::error::release_evidence: {key!r} is not a declared required cell") + return 1 + spec = EmitSpec( + out_root=args.out_root, + release_sha=args.release_sha, + workflow_name=args.workflow_name, + run_id=args.run_id, + run_attempt=args.run_attempt, + event=args.event, + job_id=args.job_id, + matrix_cell=args.matrix_cell, + terminal_outcome=args.terminal_outcome, + runner_identity=_optional(args.runner_identity), + chrome_identity=_optional(args.chrome_identity), + chrome_launched=args.chrome_launched, + junit=_optional(args.junit), + artifacts=_artifact_pairs(args.artifact), + mq_ids=tuple(args.mq), + ) + record = build_record(spec) + out = args.out_root / args.release_sha / args.job_id / f"{args.matrix_cell}.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(dumps(record), encoding="utf-8", newline="\n") + problems = validate_record(record, expect_key=key) + print(f"wrote {out}") + for problem in problems: + print(f"::error::release_evidence: {problem}") + return 1 if problems else 0 + + +def _cmd_validate(args: argparse.Namespace) -> int: + record = json.loads(args.record.read_text(encoding="utf-8")) + problems = validate_record(record, expect_key=args.key) + for problem in problems: + print(f"::error::release_evidence: {problem}") + if problems: + return 1 + print(f"{args.record}: valid {SCHEMA} child record for {args.key}") + return 0 + + +def _cmd_aggregate(args: argparse.Namespace) -> int: + spec = AggregateSpec( + root=args.root, + release_sha=args.release_sha, + workflow_name=args.workflow_name, + run_id=args.run_id, + run_attempt=args.run_attempt, + event=args.event, + claims_path=args.claims, + ) + aggregate, problems = build_aggregate(spec) + if aggregate: + out = args.root / args.release_sha / AGGREGATE_JOB / "aggregate.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(dumps(aggregate), encoding="utf-8", newline="\n") + print(f"wrote {out}") + for problem in problems: + print(f"::error::release_evidence: {problem}") + if problems: + print(f"::error::release_evidence: {len(problems)} ledger problem(s)") + return 1 + print(f"aggregate: all {len(REQUIRED_KEYS)} required cells validated") + return 0 + + +def _cmd_claims(args: argparse.Namespace) -> int: + claims = load_claims(args.claims) + problems = validate_claims_document(claims, registry_tool_names()) + for problem in problems: + print(f"::error::release_evidence: {problem}") + if problems: + return 1 + print(dumps(tool_surface(claims)).rstrip()) + return 0 + + +def _add_emit_parser(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser("emit", help="write this cell's child record") + parser.add_argument("--out-root", type=Path, default=Path("release-evidence")) + parser.add_argument("--release-sha", required=True) + parser.add_argument("--workflow-name", default="release-gate") + parser.add_argument("--run-id", required=True) + parser.add_argument("--run-attempt", required=True) + parser.add_argument("--event", required=True) + parser.add_argument("--job-id", required=True) + parser.add_argument("--matrix-cell", required=True) + parser.add_argument("--terminal-outcome", required=True) + parser.add_argument("--runner-identity", default="") + parser.add_argument("--chrome-identity", default="") + parser.add_argument("--chrome-launched", action="store_true") + parser.add_argument("--junit", default="") + parser.add_argument("--artifact", action="append", default=[]) + parser.add_argument("--mq", action="append", default=[]) + parser.set_defaults(func=_cmd_emit) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + _add_emit_parser(sub) + + validate = sub.add_parser("validate", help="validate one child record") + validate.add_argument("--record", type=Path, required=True) + validate.add_argument("--key", required=True) + validate.set_defaults(func=_cmd_validate) + + aggregate = sub.add_parser("aggregate", help="build the fail-closed aggregate") + aggregate.add_argument("--root", type=Path, default=Path("release-evidence")) + aggregate.add_argument("--release-sha", required=True) + aggregate.add_argument("--workflow-name", default="release-gate") + aggregate.add_argument("--run-id", required=True) + aggregate.add_argument("--run-attempt", required=True) + aggregate.add_argument("--event", required=True) + aggregate.add_argument("--claims", type=Path, default=CLAIMS_PATH) + aggregate.set_defaults(func=_cmd_aggregate) + + claims = sub.add_parser("claims", help="validate the tool claim ledger") + claims.add_argument("--claims", type=Path, default=CLAIMS_PATH) + claims.set_defaults(func=_cmd_claims) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return int(args.func(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/release_tool_claims.json b/tools/release_tool_claims.json new file mode 100644 index 0000000..2b97e70 --- /dev/null +++ b/tools/release_tool_claims.json @@ -0,0 +1,104 @@ +{ + "schema": "release-evidence/v1#tool-claims", + "note": [ + "The per-tool claim ledger read ONLY by tools/release_evidence.py (plan_RELEASE W5).", + "A tool is release-qualified ONLY by an entry in `qualified`, and every such entry is", + "re-checked against that run's release-evidence/v1 records by the `release-evidence`", + "job: the cited node must have executed AND passed on every cell it names, must not be", + "the representative journey, and a `stdio` claim must come from a job that drives the", + "real-stdio lane. A claim that cannot be verified makes the gate RED.", + "Every served tool absent from `qualified` is `served-unqualified`; its tracking id and", + "user impact come from `served_unqualified_notes` when it has a specific defect, and", + "from `default_note` otherwise. The tool NAMES are never listed here - they are derived", + "from SECTION_TOOLS so this file cannot drift from the registry." + ], + "qualified": [ + { + "tool": "set_cookie", + "outcome": "Sets a cookie the browser really holds: the value is read back over the wire and cross-checked against document.cookie, so the assertion cannot be satisfied by the tool echoing its own input.", + "transport": "stdio", + "node_id": "tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip", + "site_shape": "local fixture app served over a real http:// origin (tests/fixture_app)", + "required_cells": ["transport/Linux-X64", "transport/Windows-X64"] + }, + { + "tool": "get_cookies", + "outcome": "Returns the cookie's EXACT value - asserted from both CDP retrieval paths (scoped Network.getCookies and Network.getAllCookies), against a per-run unique value, not presence, type, or count.", + "transport": "stdio", + "node_id": "tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip", + "site_shape": "local fixture app served over a real http:// origin (tests/fixture_app)", + "required_cells": ["transport/Linux-X64", "transport/Windows-X64"] + }, + { + "tool": "clear_cookies", + "outcome": "Removes the cookie, proved by RE-READING it afterwards rather than by trusting the tool's own return value.", + "transport": "stdio", + "node_id": "tests/test_e2e_transport_cookies.py::test_real_transport_cookie_round_trip", + "site_shape": "local fixture app served over a real http:// origin (tests/fixture_app)", + "required_cells": ["transport/Linux-X64", "transport/Windows-X64"] + } + ], + "default_note": { + "tracking_id": "F-776", + "user_impact": "Tested, not proved at the wire: the E2E suite drives this tool against REAL Chrome through the in-process seam, but no per-tool assertion exists over the stdio transport a user speaks. plan_RELEASE 2.5 does not let in-process evidence license a transport claim. Not a known defect - a question of where the test runs." + }, + "served_unqualified_notes": { + "select_option": { + "tracking_id": "E8-1", + "user_impact": "A select whose page script re-declares a const swallows the change: the tool returns True while the option did not change. Silent wrong-success." + }, + "click_element": { + "tracking_id": "E8-2", + "user_impact": "Clicking a disabled control returns True. The caller cannot distinguish 'clicked' from 'ignored by the browser'." + }, + "type_text": { + "tracking_id": "E7-1/E8-3/E8-4", + "user_impact": "clear_first bypasses readonly (E8-3); contenteditable is not cleared before typing (E7-1); range/color/date inputs are not reachable through this path at all (E8-4)." + }, + "get_element_state": { + "tracking_id": "E7-6", + "user_impact": "Reports HTML attributes, not live DOM properties, so a value changed by script is reported stale." + }, + "modify_headers": { + "tracking_id": "F-165", + "user_impact": "Duplicate header names are mishandled in the rewrite loop." + }, + "switch_tab": { + "tracking_id": "F-775c-residual", + "user_impact": "Activation is fixed (FIX-F), but the instance's main tab is still stored from the raw browser.tabs entry, which can be a Connection rather than a Tab. Loud if it fires, and it seeds the F-775a family." + }, + "close_tab": { + "tracking_id": "F-775b/macOS-close-flake", + "user_impact": "Closing by target id is fixed (FIX-F). One macOS CI attempt returned True while the target survived a 10s poll; that observation is NOT reproducible and is NOT recorded as closed." + }, + "close_instance": { + "tracking_id": "F-775d", + "user_impact": "Teardown uses the verified CDP call but has no dedicated pin - it is exercised only indirectly by integration teardown, so a regression here would surface as someone else's flake." + }, + "execute_script": { + "tracking_id": "TRUST-BOUNDARY", + "user_impact": "Executes caller-supplied JavaScript in the page by design. The trust boundary is unverified: W12 has not run." + }, + "inject_and_execute_script": { + "tracking_id": "TRUST-BOUNDARY", + "user_impact": "Injects and runs caller-supplied JavaScript by design. Trust boundary unverified (W12 not run)." + }, + "call_javascript_function": { + "tracking_id": "TRUST-BOUNDARY", + "user_impact": "Invokes arbitrary page functions by design. Trust boundary unverified (W12 not run)." + }, + "execute_python_in_browser": { + "tracking_id": "TRUST-BOUNDARY", + "user_impact": "Evaluates caller-supplied Python in the server process by design. Trust boundary unverified (W12 not run)." + }, + "create_python_binding": { + "tracking_id": "TRUST-BOUNDARY", + "user_impact": "Exposes a Python callable to page JavaScript by design. Trust boundary unverified (W12 not run)." + }, + "execute_cdp_command": { + "tracking_id": "TRUST-BOUNDARY", + "user_impact": "Sends arbitrary CDP commands by design - the widest surface in the server. Trust boundary unverified (W12 not run)." + } + }, + "not_served": [] +} diff --git a/uv.lock b/uv.lock index 2b8eedf..8d636d8 100644 --- a/uv.lock +++ b/uv.lock @@ -1593,7 +1593,7 @@ wheels = [ [[package]] name = "stealth-chrome-devtools-mcp" -version = "1.2.0" +version = "2.0.0" source = { editable = "." } dependencies = [ { name = "fastmcp" },