From e0773ed9c9f223376c64f7e3818b7805ff723541 Mon Sep 17 00:00:00 2001 From: Jake Fineman Date: Thu, 3 Sep 2026 20:38:49 -0400 Subject: [PATCH 1/2] feat(ga): clean-room acceptance for what the registries actually serve CI has been testing the repository while the registries served something else. That is how every artifact regression in the pre-GA audit reached users with a green build behind it: source fixed, CI green, npm and PyPI serving a broken package for days. A green source branch cannot certify an artifact that is already published. scripts/ga/registry-cleanroom.mjs installs each published package from its PUBLIC registry into a throwaway directory or venv - never this checkout, never npm link, never pip install -e - and asserts it behaves. npm isolation is load-bearing: the user config is generated fresh per run, because a developer machine with @wave-av:registry pointed at a private registry will silently test a different artifact than customers receive. Checks: static ESM import, CJS require, every declared subpath export, CLI --help exit status, CLI --version against the installed package version, help banner consistency, first-party dependency pinning, npm provenance attestation, MCP initialize + tools/list, serverInfo.version against package version, README -advertised tools against served tools, Python documented-import success, and Python top-level names against sys.stdlib_module_names. Runs nightly as well as on release. Nightly is not decoration: a published dependency range is resolved on the day a customer installs, so an artifact can break with no commit anywhere. Emits ga-evidence.json keyed to ART-001, SUPPLY-001 and VER-001 with a fingerprint that excludes timestamps, so two runs observing the same artifacts agree. This does not add a competitor to registry-parity.yml. Parity asks whether the declared version equals the published version and never installs anything, so it cannot see a package whose version number is right and whose contents are broken. The two are complementary and that file is untouched. First run against the live registries fails, correctly, on five defects; a control run over @wave-av/sdk and @wave-av/adk passes, so the suite discriminates rather than always failing. Detail in GA-READINESS.md. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/registry-cleanroom.yml | 155 ++++++++++++++++++ GA-READINESS.md | 164 +++++++++++++++++++ README.md | 12 +- ga-out/.gitignore | 8 + scripts/ga/cleanroom-checks.mjs | 168 +++++++++++++++++++ scripts/ga/cleanroom-targets.json | 96 +++++++++++ scripts/ga/cleanroom-targets.mjs | 151 +++++++++++++++++ scripts/ga/cleanroom-util.mjs | 84 ++++++++++ scripts/ga/cleanroom_python_assert.py | 171 ++++++++++++++++++++ scripts/ga/mcp-stdio-probe.mjs | 124 ++++++++++++++ scripts/ga/registry-cleanroom.mjs | 196 +++++++++++++++++++++++ 11 files changed, 1328 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/registry-cleanroom.yml create mode 100644 GA-READINESS.md create mode 100644 ga-out/.gitignore create mode 100644 scripts/ga/cleanroom-checks.mjs create mode 100644 scripts/ga/cleanroom-targets.json create mode 100644 scripts/ga/cleanroom-targets.mjs create mode 100644 scripts/ga/cleanroom-util.mjs create mode 100644 scripts/ga/cleanroom_python_assert.py create mode 100644 scripts/ga/mcp-stdio-probe.mjs create mode 100644 scripts/ga/registry-cleanroom.mjs diff --git a/.github/workflows/registry-cleanroom.yml b/.github/workflows/registry-cleanroom.yml new file mode 100644 index 0000000..e06b0f8 --- /dev/null +++ b/.github/workflows/registry-cleanroom.yml @@ -0,0 +1,155 @@ +name: registry clean-room acceptance + +# Tests ONLY what the public registries serve — never this checkout. +# +# `registry parity` (the sibling workflow) answers "does the declared version equal the published +# version". That is necessary and not sufficient: it never installs anything, so it cannot see a +# package whose version number is right and whose CONTENTS are broken. Every artifact regression in +# the pre-GA audit was of that second kind — CI green on source while npm and PyPI served a broken +# build for days. This workflow installs the published artifact into a throwaway directory or venv +# and asserts it actually works. +# +# WHY NIGHTLY AND NOT ONLY ON RELEASE +# A published package can break with NO commit anywhere. @wave-av/cli's own published dependency +# range on @wave-av/sdk is a caret range, so what a customer receives is decided by npm's resolver +# on the day they install, not by anything in this repo. Only a scheduled run catches that. +# +# BLOCKING BEHAVIOUR +# schedule / workflow_run(release) / workflow_dispatch -> HARD FAIL, and open-or-update an issue. +# pull_request -> informational only. +# A PR is not the cause of an already-published defect, so a PR is not blocked by one. The PR run +# exists so that a change to the gate itself is exercised before it merges. There is deliberately +# NO path filter: the check reports on every PR, which is what makes it eligible to become a +# required status check later (a path-filtered required check stays permanently unreported and +# blocks every PR that misses the filter — the lesson already recorded in registry-parity.yml). + +on: + schedule: + # 09:00 UTC — deliberately offset from `registry parity` (14:00) so a registry outage does not + # take out both signals in the same window. + - cron: "0 9 * * *" + workflow_dispatch: + inputs: + versions: + description: 'Exact versions to accept, e.g. "@wave-av/cli=1.0.9,wave-sdk=2.1.0" (default: registry latest)' + type: string + required: false + only: + description: 'Comma-separated target ids to run (default: all). Ids are in scripts/ga/cleanroom-targets.json' + type: string + required: false + workflow_run: + # Runs after a real publish so a release is verified against the registry it just wrote to. + workflows: ["npm publish (OIDC + provenance)"] + types: [completed] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + cleanroom: + runs-on: ubuntu-latest + timeout-minutes: 30 + # On workflow_run, only a SUCCESSFUL publish is worth verifying; a failed publish has its own + # error and would produce a confusing second failure here. + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + permissions: + contents: read + issues: write # fail-loud: open or update the tracking issue + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Run clean-room acceptance against the public registries + id: cleanroom + env: + CLEANROOM_VERSIONS: ${{ inputs.versions }} + ONLY: ${{ inputs.only }} + run: | + set -uo pipefail + args=(--python python3 --out-dir "$GITHUB_WORKSPACE/ga-out") + [ -n "${ONLY:-}" ] && args+=(--only "$ONLY") + set +e + node scripts/ga/registry-cleanroom.mjs "${args[@]}" 2>&1 | tee "$RUNNER_TEMP/cleanroom.log" + code=${PIPESTATUS[0]} + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + { + echo "## Registry clean-room acceptance" + echo + echo "Exit code \`$code\` (0 = all checks passed, 1 = an artifact failed, 2 = the gate could not run)." + echo + echo '```' + cat "$RUNNER_TEMP/cleanroom.log" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Upload GA evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ga-evidence-registry-cleanroom + path: ga-out/ + if-no-files-found: warn + retention-days: 90 + + - name: Fail loudly — open or update the tracking issue + # A nightly that fails quietly is worse than no nightly at all. + if: steps.cleanroom.outputs.exit_code != '0' && github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + EXIT_CODE: ${{ steps.cleanroom.outputs.exit_code }} + run: | + set -euo pipefail + title="registry clean-room acceptance is failing" + # Only the failing-check lines: the issue must say WHAT is broken, not page a human to + # go read a log to find out. + summary=$(sed -n '/^REGISTRY CLEAN-ROOM FAILED/,$p' "$RUNNER_TEMP/cleanroom.log" | head -40) + [ -z "$summary" ] && summary="The gate did not complete (exit $EXIT_CODE). See the run log." + body=$(printf '%s\n\n```\n%s\n```\n\nRun: %s\nEvent: %s\n\nThe published artifacts do not satisfy ART-001 / SUPPLY-001 / VER-001. A green source branch does not certify these — the registry is what customers install.\n' \ + "Clean-room acceptance against the public registries failed." "$summary" "$RUN_URL" "$GITHUB_EVENT_NAME") + existing=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "$title in:title" --json number --jq '.[0].number // empty') + if [ -n "$existing" ]; then + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" --body "$body" + echo "updated issue #$existing" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body" \ + || echo "::warning::could not open the tracking issue; the job still fails below" + fi + + - name: Enforce + # Hard-fail everywhere the result is actionable. A pull request is informational: it did not + # publish the artifact under test and cannot fix it. + if: github.event_name != 'pull_request' + env: + CODE: ${{ steps.cleanroom.outputs.exit_code }} + run: | + if [ "$CODE" != "0" ]; then + echo "::error title=registry clean-room::published artifacts failed clean-room acceptance (exit $CODE) — see the job summary" + exit 1 + fi + echo "registry clean-room acceptance passed" + + - name: Report (pull request, informational) + if: github.event_name == 'pull_request' + env: + CODE: ${{ steps.cleanroom.outputs.exit_code }} + run: | + [ "$CODE" = "0" ] && echo "clean-room acceptance passed" \ + || echo "::warning title=registry clean-room::published artifacts fail clean-room acceptance (exit $CODE). Not blocking this PR — see the job summary." diff --git a/GA-READINESS.md b/GA-READINESS.md new file mode 100644 index 0000000..2b4b047 --- /dev/null +++ b/GA-READINESS.md @@ -0,0 +1,164 @@ +# WAVE GA compliance — wave-av/sdks + +> Criterion IDs are global and immutable. `unknown` and a waiver are **not** a pass. A green +> repository is necessary but not sufficient for platform GA. + +## Repository declaration + +```yaml +repository: wave-av/sdks +revision: 70b2a04a205658a25b3723adec9faf665e204af6 +repo_class: sdk +owner: WAVE platform / SDK publishing +last_evaluated_at: 2026-09-04T00:31:56Z +evaluator_version: scripts/ga/registry-cleanroom.mjs +spec_version: 1.0.0 +overall_status: fail +``` + +`overall_status: fail` is the honest reading, and it is an improvement on what came before it: +these criteria were **unknown** until this repository could install what the registries serve and +look. The gate now produces evidence. The evidence says the published artifacts are broken. + +## How this repo proves compliance + +`scripts/ga/registry-cleanroom.mjs` installs each published artifact from its **public registry** +into a throwaway directory or venv — never from this checkout, never `npm link`, never +`pip install -e` — and asserts it behaves. It emits `ga-out/ga-evidence.json` keyed to criterion +IDs, plus `ga-out/cleanroom-report.json` with per-check detail and artifact digests. The run +output is a CI artifact and is never committed; a committed report would let a stale file +masquerade as current evidence. + +The distinction from the sibling `registry parity` workflow matters. Parity asks *"does the +declared version equal the published version"* and never installs anything, so it cannot see a +package whose version number is correct and whose contents are broken. Every artifact regression in +the pre-GA audit was of that second kind. + +```bash +# everything the registries currently serve +node scripts/ga/registry-cleanroom.mjs + +# a specific release, pinned rather than whatever `latest` points at +node scripts/ga/registry-cleanroom.mjs --versions '@wave-av/cli=1.0.9,wave-sdk=2.1.0' +``` + +Exit `0` all checks passed · `1` an artifact failed · `2` the gate could not run. Exit 2 is never +to be read as a pass. + +Schedule: nightly at 09:00 UTC, after every successful npm publish, on demand, and informationally +on every pull request (`.github/workflows/registry-cleanroom.yml`). Nightly is not decoration — a +published package can break with no commit anywhere, because a published dependency **range** is +resolved on the day a customer installs. + +| Criterion | Title | GA must-pass | Repo status | Evidence | +|---|---|---:|---|---| +| ART-001 | Published artifacts install, import, start, identify themselves, and match source | true | **fail** | `ga-out/ga-evidence.json` (evaluator: `scripts/ga/registry-cleanroom.mjs`) | +| SUPPLY-001 | Builds have provenance, signatures, SBOMs, dependency policy, and protected release identity | true | **fail** (partial coverage — see note) | `ga-out/ga-evidence.json` | +| VER-001 | Version and release truth agree from source through deployment | true | **fail** (registry half) | `ga-out/ga-evidence.json` | +| CONTRACT-001 | One promoted contract is the source of truth across spec, gateway, registry, MCP, SDK and CLI | true | unknown | not evaluated by this repo | +| COMPAT-001 | Backward compatibility, versioning, deprecation, and sunset policy are enforced | true | unknown | not evaluated by this repo | +| DX-001 | A new developer can complete one honest golden path from published materials | true | unknown | not evaluated by this repo | +| STATUS-001 | Marketing, registry, preview labels, availability, and status tell the same truth | true | unknown | not evaluated by this repo | + +Criteria absent from this table are owned by other repositories and surfaces; this repository makes +no claim about them. Absence here is `unknown` at the platform gate, not `not_applicable`. + +## Per-criterion evidence + +```yaml +criterion_id: ART-001 +status: fail +owner: WAVE platform / SDK publishing +verification_command: node scripts/ga/registry-cleanroom.mjs +verified_revision: 70b2a04a205658a25b3723adec9faf665e204af6 +verified_at: 2026-09-04T00:31:56Z +evidence: + - uri: ci://wave-av/sdks/.github/workflows/registry-cleanroom.yml#cleanroom-report.json + sha256: 0f03603df2bc33f50958a75c8998777463507c1f9e3ee0fec811d82a3ca1206d +pass_condition_from_spec: > + All supported runtimes pass from public registries; Python uses a non-stdlib-colliding import; + source, package metadata, CLI banner, tag and GitHub release agree. +notes: | + Observed against the live registries on 2026-09-04. Passing: + @wave-av/sdk@2.1.3 ESM import, CJS require, all 46 declared subpath exports resolve. + @wave-av/adk@1.0.15 installs and imports. + @wave-av/mcp-server@0.2.0 starts over stdio, lists 18 tools, serves every tool its shipped + README advertises. + Failing: + @wave-av/cli@1.0.8 `wave --version` prints 1.0.0. Installs cleanly and `--help` exits 0, + so nothing short of running the binary detects this. + wave-sdk@2.0.0 (PyPI) `from wave_sdk import Wave` raises ModuleNotFoundError. The wheel's + only top-level name is `wave`, which collides with the CPython stdlib + module of that name; because the stdlib directory precedes + site-packages, `import wave` returns the stdlib WAV reader and the SDK + is unreachable by any name. The artifact is unusable as published. + wave-av-sdk@2.0.0 (PyPI) identical defect. +``` + +```yaml +criterion_id: VER-001 +status: fail +owner: WAVE platform / SDK publishing +verification_command: node scripts/ga/registry-cleanroom.mjs +verified_revision: 70b2a04a205658a25b3723adec9faf665e204af6 +verified_at: 2026-09-04T00:31:56Z +evidence: + - uri: ci://wave-av/sdks/.github/workflows/registry-cleanroom.yml#cleanroom-report.json + sha256: 0f03603df2bc33f50958a75c8998777463507c1f9e3ee0fec811d82a3ca1206d +pass_condition_from_spec: > + Every shipped component resolves to one source revision and version; no newer source is + represented as deployed; mutable channels are labeled; deployment receipt identifies artifact + digest. +notes: | + This repository covers the REGISTRY half of VER-001 — does the artifact agree with itself about + which build it is. Two published artifacts do not: + @wave-av/cli@1.0.8 binary self-reports 1.0.0 + @wave-av/mcp-server@0.2.0 serverInfo.version reports 0.1.0 + In both cases the package metadata is correct and the code inside it disagrees, so a version + comparison against the registry cannot see the defect — only running the artifact can. + Tag/GitHub-release/deployed-endpoint agreement is NOT covered here and remains unknown; it belongs + to the release-ledger check named in the spec's runnable_command. +``` + +```yaml +criterion_id: SUPPLY-001 +status: fail +owner: WAVE platform / SDK publishing +verification_command: node scripts/ga/registry-cleanroom.mjs +verified_revision: 70b2a04a205658a25b3723adec9faf665e204af6 +verified_at: 2026-09-04T00:31:56Z +evidence: + - uri: ci://wave-av/sdks/.github/workflows/registry-cleanroom.yml#cleanroom-report.json + sha256: 0f03603df2bc33f50958a75c8998777463507c1f9e3ee0fec811d82a3ca1206d +pass_condition_from_spec: > + Release artifacts are built by approved CI from an immutable source revision, provenance is + verifiable, SBOM is attached, critical known vulnerabilities are resolved or explicitly + risk-accepted, and publisher accounts require strong MFA. +notes: | + PARTIAL COVERAGE — this evaluator checks two of the five clauses. A `fail` here is therefore + sound, but a future `pass` would NOT be sufficient to pass SUPPLY-001 on its own. + Covered and failing: + provenance @wave-av/cli@1.0.8 carries no npm provenance attestation (dist.attestations is + null), while sdk, mcp-server and adk each carry a + https://slsa.dev/provenance/v1 attestation. The CLI was published outside the + provenance-emitting pipeline and cannot be traced to an approved CI build. + dependency @wave-av/cli@1.0.8 declares `@wave-av/sdk: "^2.0.11"`. What a customer receives + policy is decided by npm's resolver on the day they install; today that is 2.1.3. The + published artifact is not reproducible, and this is the precise mechanism by + which a broken SDK shipped inside a CLI that no one had changed. + NOT covered here, still unknown: SBOM attachment, vulnerability posture, publisher MFA and + branch-protection attestation. +``` + +## Operator actions to finish these criteria + +1. **Make the release gate blocking.** The clean-room job hard-fails on schedule, release and + dispatch, but it is not yet a *required* status check. Add + `registry clean-room acceptance / cleanroom` to the default branch's required checks. It runs on + every pull request with no path filter precisely so it can be made required without going + permanently unreported. +2. **Fix the three artifact defects the gate found** (each needs a publish, which is a named floor + and not this lane's to cross): the CLI version constant, the MCP server's `serverInfo.version`, + and the Python distribution's top-level module name. +3. **Republish the CLI through the provenance-emitting workflow** so `dist.attestations` is + populated, and exact-pin its first-party dependency. diff --git a/README.md b/README.md index 49c682a..70cf198 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,20 @@ Agents and short-lived sandboxes can skip the SDK entirely and call the gateway sdk-typescript/packages/{core,clips,sdk}/ # the TS fleet (pnpm workspace) scripts/sync-from-monorepo.sh # allowlist sync + fail-closed secret scan scripts/check-registry-parity.py # declared-vs-published gate (vendored from wave-foundation) +scripts/ga/registry-cleanroom.mjs # clean-room acceptance of what the REGISTRIES serve .github/workflows/publish-npm.yml # OIDC trusted publishing + provenance -.github/workflows/registry-parity.yml # parity gate +.github/workflows/registry-parity.yml # parity gate (declared version == published version) +.github/workflows/registry-cleanroom.yml # clean-room gate (published artifact actually works) ``` +## GA readiness + +This repository participates in the WAVE GA gate. Its criterion ownership, runnable checks and latest evidence are in [GA-READINESS.md](./GA-READINESS.md). CI emits `ga-evidence.json` against spec version 1.0.0. Stable criterion IDs are never renamed or reused. + +The check that matters here is **clean-room acceptance**: `scripts/ga/registry-cleanroom.mjs` installs each package from its public registry into a throwaway directory or venv — never from this checkout — and asserts the artifact imports, starts, and reports its own version honestly. A green source branch cannot certify a package already on npm or PyPI, which is why this runs nightly as well as on release: a published dependency *range* is resolved on the day a customer installs, so an artifact can break with no commit anywhere. + +A green repository is necessary but not sufficient for platform GA. The platform release is **NO-GO** unless the root gate aggregator has current passing evidence for every must-pass criterion across all owning repositories and deployed surfaces. `unknown`, stale evidence, missing evidence, a waiver, or an unapproved `not_applicable` counts as failure. + ## License Apache-2.0 © WAVE — see [LICENSE](LICENSE) and [NOTICE](NOTICE). diff --git a/ga-out/.gitignore b/ga-out/.gitignore new file mode 100644 index 0000000..1dc2cff --- /dev/null +++ b/ga-out/.gitignore @@ -0,0 +1,8 @@ +# Clean-room run output (cleanroom-report.json, ga-evidence.json) is a CI artifact, never a +# committed one: evidence must describe a run that actually happened, in the environment that +# happened to run it. Committing a locally generated report would let a stale file masquerade as +# current evidence — the exact failure mode the GA gate exists to prevent. +# +# Self-ignoring so the root .gitignore does not need to change. +* +!.gitignore diff --git a/scripts/ga/cleanroom-checks.mjs b/scripts/ga/cleanroom-checks.mjs new file mode 100644 index 0000000..dac62ea --- /dev/null +++ b/scripts/ga/cleanroom-checks.mjs @@ -0,0 +1,168 @@ +// The acceptance checks themselves. Each takes a per-target context and returns one result. +// +// Design rule: a check reports the artifact's OWN words back (its version strings, its tool list, +// its resolved module paths) rather than asserting a value hardcoded here. A suite that encodes +// the expected answer drifts into agreeing with itself; a suite that quotes the artifact can only +// pass when the artifact is actually coherent. + +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { bad, installedManifest, ok, run } from './cleanroom-util.mjs'; + +const DEFAULT_ADVERTISED_TOOL_PATTERN = '(?:wave|mvp)_[a-z0-9_]+'; + +export const CHECKS = { + 'npm-provenance-attested': async (ctx) => { + const att = ctx.packument?.dist?.attestations; + if (att?.provenance?.predicateType) { + return ok('npm-provenance-attested', `provenance attestation present (${att.provenance.predicateType})`); + } + return bad('npm-provenance-attested', + `${ctx.pkg}@${ctx.version} has NO provenance attestation on npm (dist.attestations=${JSON.stringify(att ?? null)}). ` + + 'The published artifact cannot be traced to an approved CI build of an immutable source revision.'); + }, + + 'esm-import': async (ctx) => { + const f = join(ctx.room, 'probe-esm.mjs'); + writeFileSync(f, `import * as m from ${JSON.stringify(ctx.pkg)};\nconsole.log(Object.keys(m).length);\n`); + const r = run(process.execPath, [f], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + return r.status === 0 + ? ok('esm-import', `static ESM \`import * from '${ctx.pkg}'\` OK (${r.stdout.trim()} named exports)`) + : bad('esm-import', `static ESM import failed: ${(r.stderr || r.stdout).trim().slice(0, 400)}`); + }, + + 'cjs-require': async (ctx) => { + const f = join(ctx.room, 'probe-cjs.cjs'); + writeFileSync(f, `const m = require(${JSON.stringify(ctx.pkg)});\nconsole.log(Object.keys(m).length);\n`); + const r = run(process.execPath, [f], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + return r.status === 0 + ? ok('cjs-require', `\`require('${ctx.pkg}')\` OK (${r.stdout.trim()} keys)`) + : bad('cjs-require', `CJS require failed: ${(r.stderr || r.stdout).trim().slice(0, 400)}`); + }, + + 'subpath-exports': async (ctx) => { + const exportsMap = ctx.manifest?.exports; + if (!exportsMap || typeof exportsMap !== 'object') { + return bad('subpath-exports', `${ctx.pkg}@${ctx.version} declares no "exports" map — subpath resolution is unverifiable`); + } + const subpaths = Object.keys(exportsMap).filter((k) => k !== './package.json'); + const f = join(ctx.room, 'probe-subpaths.mjs'); + writeFileSync(f, [ + `const subpaths = ${JSON.stringify(subpaths)};`, + `const pkg = ${JSON.stringify(ctx.pkg)};`, + 'const failed = [];', + 'for (const sp of subpaths) {', + " const spec = sp === '.' ? pkg : pkg + '/' + sp.replace(/^\\.\\//, '');", + " try { await import(spec); } catch (e) { failed.push(spec + ': ' + (e.code || e.name)); }", + '}', + 'console.log(JSON.stringify({ total: subpaths.length, failed }));', + ].join('\n')); + const r = run(process.execPath, [f], { cwd: ctx.room, env: ctx.env, timeout: 180000 }); + if (r.status !== 0) return bad('subpath-exports', `subpath probe crashed: ${(r.stderr || r.stdout).trim().slice(0, 400)}`); + let parsed; + try { parsed = JSON.parse(r.stdout.trim().split('\n').pop()); } + catch { return bad('subpath-exports', `unparseable probe output: ${r.stdout.slice(0, 200)}`); } + return parsed.failed.length === 0 + ? ok('subpath-exports', `all ${parsed.total} declared subpath exports resolve`) + : bad('subpath-exports', `${parsed.failed.length}/${parsed.total} declared subpath exports fail to resolve: ${parsed.failed.slice(0, 8).join('; ')}`); + }, + + 'bin-help-exit-zero': async (ctx) => { + const bin = ctx.binPath(); + if (!bin) return bad('bin-help-exit-zero', `no bin found for ${ctx.pkg} (declared bin: ${JSON.stringify(ctx.manifest?.bin ?? null)})`); + const r = run(process.execPath, [bin, '--help'], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + return r.status === 0 + ? ok('bin-help-exit-zero', `\`${ctx.target.bin} --help\` exited 0 (${r.stdout.split('\n').length} lines)`) + : bad('bin-help-exit-zero', `\`${ctx.target.bin} --help\` exited ${r.status}: ${(r.stderr || r.stdout).trim().slice(0, 400)}`); + }, + + 'bin-version-matches-package': async (ctx) => { + const bin = ctx.binPath(); + if (!bin) return bad('bin-version-matches-package', `no bin found for ${ctx.pkg}`); + const r = run(process.execPath, [bin, '--version'], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + if (r.status !== 0) return bad('bin-version-matches-package', `\`${ctx.target.bin} --version\` exited ${r.status}: ${(r.stderr || r.stdout).trim().slice(0, 300)}`); + const printed = (r.stdout.trim().match(/\d+\.\d+\.\d+[^\s]*/) || [r.stdout.trim()])[0]; + return printed === ctx.manifest.version + ? ok('bin-version-matches-package', `\`${ctx.target.bin} --version\` prints ${printed}, matching the installed package version`) + : bad('bin-version-matches-package', + `VERSION LIE: npm served ${ctx.pkg}@${ctx.manifest.version} but \`${ctx.target.bin} --version\` prints ${printed}. ` + + 'A user cannot tell which build they are running, and a bug report cannot be tied to a revision.'); + }, + + 'bin-help-banner-version-consistent': async (ctx) => { + const bin = ctx.binPath(); + if (!bin) return bad('bin-help-banner-version-consistent', `no bin found for ${ctx.pkg}`); + const r = run(process.execPath, [bin, '--help'], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + // Only lines that actually advertise a version count. A bare semver elsewhere in help text + // (an example payload, a protocol number) must not manufacture a false failure. + const claimed = new Set(); + for (const line of `${r.stdout}\n${r.stderr}`.split('\n')) { + if (!/version|\bv\d/i.test(line)) continue; + for (const m of line.matchAll(/\bv?(\d+\.\d+\.\d+)\b/g)) claimed.add(m[1]); + } + if (claimed.size === 0) return ok('bin-help-banner-version-consistent', 'help output advertises no version string (nothing to contradict)'); + const wrong = [...claimed].filter((v) => v !== ctx.manifest.version); + return wrong.length === 0 + ? ok('bin-help-banner-version-consistent', `help banner advertises ${[...claimed].join(', ')}, matching the installed version`) + : bad('bin-help-banner-version-consistent', `help banner advertises version(s) ${wrong.join(', ')} but npm served ${ctx.manifest.version}`); + }, + + 'declared-dep-ranges-pinned': async (ctx) => { + // A floating range on a first-party sibling put a broken SDK inside a "known good" CLI with + // no commit anywhere. A published artifact whose own dependencies can move is not reproducible, + // which is why this must be checked nightly and not only at release. + const firstParty = Object.entries(ctx.manifest?.dependencies || {}).filter(([n]) => n.startsWith('@wave-av/')); + if (firstParty.length === 0) return ok('declared-dep-ranges-pinned', 'no first-party runtime dependencies to pin'); + const resolved = firstParty.map(([n]) => `${n}@${installedManifest(ctx.room, n)?.version ?? ''}`); + const floating = firstParty.filter(([, range]) => !/^\d+\.\d+\.\d+/.test(range)); + return floating.length === 0 + ? ok('declared-dep-ranges-pinned', `first-party deps exact-pinned; resolved to ${resolved.join(', ')}`) + : bad('declared-dep-ranges-pinned', + `${ctx.pkg}@${ctx.manifest.version} declares floating first-party range(s) ` + + `${floating.map(([n, r]) => `${n}: "${r}"`).join(', ')} — today they resolve to ${resolved.join(', ')}, ` + + 'but tomorrow the same published artifact can ship a different sibling with no commit anywhere.'); + }, + + 'mcp-server-lists-tools': async (ctx) => { + const probe = await ctx.mcpProbe(); + if (!probe.ok) return bad('mcp-server-lists-tools', `server did not complete initialize + tools/list: ${probe.error}`); + return probe.toolCount > 0 + ? ok('mcp-server-lists-tools', `server started and listed ${probe.toolCount} tools: ${probe.toolNames.slice(0, 6).join(', ')}${probe.toolCount > 6 ? ', …' : ''}`) + : bad('mcp-server-lists-tools', 'server started but advertises zero tools'); + }, + + 'mcp-serverinfo-version-matches-package': async (ctx) => { + const probe = await ctx.mcpProbe(); + if (!probe.ok) return bad('mcp-serverinfo-version-matches-package', `could not reach serverInfo: ${probe.error}`); + const reported = probe.serverInfo?.version ?? null; + return reported === ctx.manifest.version + ? ok('mcp-serverinfo-version-matches-package', `serverInfo.version ${reported} matches the installed package version`) + : bad('mcp-serverinfo-version-matches-package', + `VERSION LIE: npm served ${ctx.pkg}@${ctx.manifest.version} but the running server reports ` + + `serverInfo.version=${JSON.stringify(reported)}. An MCP client cannot identify the build it connected to.`); + }, + + 'mcp-advertised-tools-are-served': async (ctx) => { + const readmePath = join(ctx.room, 'node_modules', ...ctx.pkg.split('/'), 'README.md'); + if (!existsSync(readmePath)) { + return bad('mcp-advertised-tools-are-served', `${ctx.pkg}@${ctx.version} ships no README.md — its advertised tool surface cannot be verified against what it serves`); + } + const probe = await ctx.mcpProbe(); + if (!probe.ok) return bad('mcp-advertised-tools-are-served', `could not list served tools: ${probe.error}`); + // Only backticked identifiers count. Bare prose matching picks up things like the API-key + // example `wave_live_...` and would fabricate a failure. + const pattern = new RegExp('`(' + (ctx.target.advertised_tool_pattern || DEFAULT_ADVERTISED_TOOL_PATTERN) + ')`', 'g'); + const advertised = [...new Set([...readFileSync(readmePath, 'utf8').matchAll(pattern)].map((m) => m[1]))].sort(); + if (advertised.length === 0) return bad('mcp-advertised-tools-are-served', 'shipped README advertises no tool names — the artifact documents nothing it serves'); + const served = new Set(probe.toolNames); + const missing = advertised.filter((t) => !served.has(t)); + if (missing.length > 0) { + return bad('mcp-advertised-tools-are-served', `shipped README advertises ${missing.length} tool(s) the server does not serve: ${missing.join(', ')}`); + } + const undocumented = probe.toolNames.filter((t) => !advertised.includes(t)); + const extra = undocumented.length + ? ` (${undocumented.length} served but undocumented: ${undocumented.slice(0, 5).join(', ')}${undocumented.length > 5 ? ', …' : ''})` + : ''; + return ok('mcp-advertised-tools-are-served', `all ${advertised.length} README-advertised tools are served${extra}`); + }, +}; diff --git a/scripts/ga/cleanroom-targets.json b/scripts/ga/cleanroom-targets.json new file mode 100644 index 0000000..e1f99c2 --- /dev/null +++ b/scripts/ga/cleanroom-targets.json @@ -0,0 +1,96 @@ +{ + "$comment": [ + "Declarative target manifest for scripts/ga/registry-cleanroom.mjs.", + "Every entry describes an artifact THE PUBLIC REGISTRY SERVES — never a local checkout.", + "`checks` names are implemented in registry-cleanroom.mjs (CHECKS table); each check", + "maps to the GA criterion IDs it produces evidence for. Adding a package here is the", + "only edit needed to bring it under the clean-room gate." + ], + "spec_version": "1.0.0", + "repository": "wave-av/sdks", + "targets": [ + { + "id": "npm-sdk", + "ecosystem": "npm", + "package": "@wave-av/sdk", + "checks": [ + "npm-provenance-attested", + "esm-import", + "cjs-require", + "subpath-exports" + ] + }, + { + "id": "npm-cli", + "ecosystem": "npm", + "package": "@wave-av/cli", + "bin": "wave", + "checks": [ + "npm-provenance-attested", + "bin-help-exit-zero", + "bin-version-matches-package", + "bin-help-banner-version-consistent", + "declared-dep-ranges-pinned" + ] + }, + { + "id": "npm-mcp-server", + "ecosystem": "npm", + "package": "@wave-av/mcp-server", + "bin": "wave-mcp-server", + "checks": [ + "npm-provenance-attested", + "esm-import", + "mcp-server-lists-tools", + "mcp-serverinfo-version-matches-package", + "mcp-advertised-tools-are-served" + ] + }, + { + "id": "npm-adk", + "ecosystem": "npm", + "package": "@wave-av/adk", + "checks": [ + "npm-provenance-attested", + "esm-import" + ] + }, + { + "id": "pypi-wave-sdk", + "ecosystem": "pypi", + "package": "wave-sdk", + "import_module": "wave_sdk", + "import_symbol": "Wave", + "checks": [ + "py-import-module", + "py-no-stdlib-shadow" + ] + }, + { + "id": "pypi-wave-av-sdk", + "ecosystem": "pypi", + "package": "wave-av-sdk", + "import_module": "wave_sdk", + "import_symbol": "Wave", + "checks": [ + "py-import-module", + "py-no-stdlib-shadow" + ] + } + ], + "criteria_map": { + "npm-provenance-attested": ["SUPPLY-001"], + "esm-import": ["ART-001"], + "cjs-require": ["ART-001"], + "subpath-exports": ["ART-001"], + "bin-help-exit-zero": ["ART-001"], + "bin-version-matches-package": ["ART-001", "VER-001"], + "bin-help-banner-version-consistent": ["ART-001", "VER-001"], + "declared-dep-ranges-pinned": ["SUPPLY-001", "VER-001"], + "mcp-server-lists-tools": ["ART-001"], + "mcp-serverinfo-version-matches-package": ["ART-001", "VER-001"], + "mcp-advertised-tools-are-served": ["ART-001"], + "py-import-module": ["ART-001"], + "py-no-stdlib-shadow": ["ART-001"] + } +} diff --git a/scripts/ga/cleanroom-targets.mjs b/scripts/ga/cleanroom-targets.mjs new file mode 100644 index 0000000..4d1f07b --- /dev/null +++ b/scripts/ga/cleanroom-targets.mjs @@ -0,0 +1,151 @@ +// Per-ecosystem target runners: stand up the clean room, install the published artifact, then +// hand a context to the checks. Nothing here reads the repository checkout. + +import { createHash } from 'node:crypto'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { CHECKS } from './cleanroom-checks.mjs'; +import { + PUBLIC_NPM, bad, fetchJson, installedFile, installedManifest, npmCleanRoom, npmEncode, ok, run, +} from './cleanroom-util.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +export async function runNpmTarget(target, args) { + const pkg = target.package; + const latest = await fetchJson(`${PUBLIC_NPM}/${npmEncode(pkg)}/latest`); + const version = args.versions[pkg] || latest.version; + const packument = version === latest.version ? latest : await fetchJson(`${PUBLIC_NPM}/${npmEncode(pkg)}/${version}`); + + const result = { + id: target.id, + ecosystem: 'npm', + package: pkg, + version, + registry: PUBLIC_NPM, + integrity: packument?.dist?.integrity || null, + resolved_from: args.versions[pkg] ? 'explicit --versions pin' : 'registry dist-tag `latest`', + checks: [], + }; + + const cr = npmCleanRoom(pkg, version); + result.clean_room = cr.room; + if (cr.failed) { + result.checks.push(bad('install', `${cr.failed}: ${(cr.install.stderr || cr.install.stdout || cr.install.error || '').trim().slice(0, 600)}`)); + return result; + } + const manifest = installedManifest(cr.room, pkg); + if (!manifest) { + result.checks.push(bad('install', `install reported success but ${pkg} is absent from node_modules`)); + return result; + } + result.installed_version = manifest.version; + result.checks.push(ok('install', `installed ${pkg}@${manifest.version} from ${PUBLIC_NPM} into a clean directory`)); + if (manifest.version !== version) { + result.checks.push(bad('install-version-matches-request', `requested ${version} but node_modules contains ${manifest.version}`)); + } + + let mcpCache = null; + const ctx = { + target, pkg, version, manifest, packument, room: cr.room, env: cr.env, + binPath() { + const b = manifest.bin; + const rel = typeof b === 'string' ? b : b && (b[target.bin] || Object.values(b)[0]); + return rel ? installedFile(cr.room, pkg, ...rel.split('/')) : null; + }, + async mcpProbe() { + if (mcpCache) return mcpCache; + const entry = ctx.binPath(); + if (!entry) { mcpCache = { ok: false, error: `no server entrypoint for ${pkg}` }; return mcpCache; } + const r = run(process.execPath, [join(HERE, 'mcp-stdio-probe.mjs'), entry], { cwd: cr.room, env: cr.env, timeout: 120000 }); + const line = r.stdout.trim().split('\n').filter(Boolean).pop(); + try { mcpCache = JSON.parse(line); } + catch { mcpCache = { ok: false, error: `unparseable probe output (exit ${r.status}): ${(r.stdout + r.stderr).slice(0, 300)}` }; } + return mcpCache; + }, + }; + + for (const name of target.checks) { + const fn = CHECKS[name]; + if (!fn) { result.checks.push(bad(name, 'check not implemented in cleanroom-checks.mjs')); continue; } + try { result.checks.push(await fn(ctx)); } + catch (e) { result.checks.push(bad(name, `check threw ${e?.name}: ${String(e?.message).slice(0, 300)}`)); } + } + return result; +} + +export async function runPypiTarget(target, args) { + const name = target.package; + const meta = await fetchJson(`https://pypi.org/pypi/${encodeURIComponent(name)}/json`); + const version = args.versions[name] || meta.info.version; + const files = version === meta.info.version + ? meta.urls + : (await fetchJson(`https://pypi.org/pypi/${encodeURIComponent(name)}/${encodeURIComponent(version)}/json`)).urls; + + const result = { + id: target.id, + ecosystem: 'pypi', + package: name, + version, + registry: 'https://pypi.org', + resolved_from: args.versions[name] ? 'explicit --versions pin' : 'PyPI info.version', + checks: [], + }; + + const wheel = files.find((f) => f.packagetype === 'bdist_wheel') || files.find((f) => f.packagetype === 'sdist'); + if (!wheel) { result.checks.push(bad('download', `PyPI serves no wheel or sdist for ${name}@${version}`)); return result; } + + const room = mkdtempSync(join(tmpdir(), 'wave-cleanroom-py-')); + result.clean_room = room; + const wheelPath = join(room, wheel.filename); // pip rejects a renamed wheel — keep the real filename + const bytes = Buffer.from(await (await fetch(wheel.url)).arrayBuffer()); + writeFileSync(wheelPath, bytes); + const sha = createHash('sha256').update(bytes).digest('hex'); + result.artifact = { filename: wheel.filename, url: wheel.url, sha256: sha }; + if (wheel.digests?.sha256 && wheel.digests.sha256 !== sha) { + result.checks.push(bad('download', `downloaded ${wheel.filename} sha256 ${sha} != PyPI-declared ${wheel.digests.sha256}`)); + return result; + } + result.checks.push(ok('download', `downloaded ${wheel.filename} from PyPI, sha256 ${sha} matches the declared digest`)); + + const venv = join(room, 'venv'); + const mk = run(args.python, ['-m', 'venv', venv], { cwd: room, timeout: 300000 }); + if (mk.status !== 0) { + result.checks.push(bad('venv', `could not create a clean venv with ${args.python}: ${(mk.stderr || mk.stdout || mk.error || '').trim().slice(0, 400)}`)); + return result; + } + const py = join(venv, 'bin', 'python'); + const inst = run(py, ['-m', 'pip', 'install', '-q', '--disable-pip-version-check', '--no-input', wheelPath], { cwd: room, timeout: 600000 }); + if (inst.status !== 0) { + result.checks.push(bad('install', `pip install of the downloaded wheel failed: ${(inst.stderr || inst.stdout).trim().slice(0, 600)}`)); + return result; + } + result.checks.push(ok('install', `pip installed the downloaded wheel into a fresh venv (${args.python})`)); + + // cwd is the throwaway room, never the repo: a checkout on sys.path could satisfy an import the + // published wheel is supposed to satisfy — exactly the illusion this suite exists to destroy. + // cleanroom_python_assert.py re-verifies that independently and reports it as its own check. + const argv = [join(HERE, 'cleanroom_python_assert.py'), '--dist', name, '--module', target.import_module]; + if (target.import_symbol) argv.push('--symbol', target.import_symbol); + const probe = run(py, argv, { cwd: room, timeout: 180000 }); + let parsed; + try { parsed = JSON.parse(probe.stdout.trim().split('\n').filter(Boolean).pop()); } + catch { + result.checks.push(bad('py-probe', `assertion script produced no parseable JSON (exit ${probe.status}): ${(probe.stdout + probe.stderr).slice(0, 400)}`)); + return result; + } + + result.python = parsed.python; + result.top_level = parsed.top_level; + const wanted = new Set(target.checks); + for (const c of parsed.checks) { + if (c.name === 'cleanroom-isolation' || wanted.has(c.name)) result.checks.push(c); + } + for (const want of target.checks) { + if (!parsed.checks.some((c) => c.name === want)) result.checks.push(bad(want, 'check not produced by cleanroom_python_assert.py')); + } + return result; +} diff --git a/scripts/ga/cleanroom-util.mjs b/scripts/ga/cleanroom-util.mjs new file mode 100644 index 0000000..473ede7 --- /dev/null +++ b/scripts/ga/cleanroom-util.mjs @@ -0,0 +1,84 @@ +// Shared primitives for the registry clean-room suite: process execution, registry fetches, +// check-result constructors, and the isolated-npm environment. +// +// The npm isolation is the load-bearing part. On a developer machine `@wave-av:registry` often +// points at a private GitHub registry, so an install that *looks* like "from npm" can quietly +// exercise a different artifact than customers receive. Every npm invocation here runs against a +// freshly generated user-config with no auth, no ambient scope override, and a private cache. + +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +export const PUBLIC_NPM = 'https://registry.npmjs.org'; + +export function run(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { + encoding: 'utf8', + timeout: opts.timeout ?? 600000, + maxBuffer: 32 * 1024 * 1024, + ...opts, + }); + return { + status: r.status, + signal: r.signal, + stdout: r.stdout || '', + stderr: r.stderr || '', + error: r.error ? String(r.error.message) : null, + }; +} + +export async function fetchJson(url) { + const res = await fetch(url, { headers: { 'user-agent': 'wave-ga-registry-cleanroom/1.0' } }); + if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`); + return res.json(); +} + +export function ok(name, detail) { return { name, ok: true, detail }; } +export function bad(name, detail) { return { name, ok: false, detail }; } + +export function npmEncode(pkg) { return pkg.replace('/', '%2f'); } + +/** Install one package from the PUBLIC npm registry into a throwaway directory. */ +export function npmCleanRoom(pkgName, version) { + const room = mkdtempSync(join(tmpdir(), 'wave-cleanroom-npm-')); + const userConfig = join(room, 'npm-userconfig'); + const globalConfig = join(room, 'npm-globalconfig'); + writeFileSync(userConfig, [ + `registry=${PUBLIC_NPM}/`, + `@wave-av:registry=${PUBLIC_NPM}/`, + 'audit=false', + 'fund=false', + 'update-notifier=false', + '', + ].join('\n')); + writeFileSync(globalConfig, ''); // npm refuses the same path for user and global config + + const env = { + ...process.env, + npm_config_userconfig: userConfig, + npm_config_globalconfig: globalConfig, + npm_config_cache: join(room, 'npm-cache'), + npm_config_registry: `${PUBLIC_NPM}/`, + NO_UPDATE_NOTIFIER: '1', + }; + + // Lifecycle scripts stay ENABLED on purpose: a customer's `npm install` runs them, so an + // artifact whose postinstall breaks is broken in the field and this suite must see it. + const init = run('npm', ['init', '-y'], { cwd: room, env }); + if (init.status !== 0) return { room, env, install: init, failed: 'npm init failed' }; + + const install = run('npm', ['install', '--no-audit', '--no-fund', '--loglevel=error', `${pkgName}@${version}`], { cwd: room, env }); + return { room, env, install, failed: install.status === 0 ? null : 'npm install failed' }; +} + +export function installedManifest(room, pkgName) { + const p = join(room, 'node_modules', ...pkgName.split('/'), 'package.json'); + return existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null; +} + +export function installedFile(room, pkgName, ...rel) { + const p = join(room, 'node_modules', ...pkgName.split('/'), ...rel); + return existsSync(p) ? p : null; +} diff --git a/scripts/ga/cleanroom_python_assert.py b/scripts/ga/cleanroom_python_assert.py new file mode 100644 index 0000000..de0345b --- /dev/null +++ b/scripts/ga/cleanroom_python_assert.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Clean-room assertions for a PyPI artifact, run INSIDE a throwaway venv. + +Invoked by scripts/ga/registry-cleanroom.mjs as: + /bin/python cleanroom_python_assert.py --dist wave-sdk --module wave_sdk --symbol Wave + +It must be run with a cwd that contains no checkout of this repository, so that a source +tree can never satisfy an import the published wheel is supposed to satisfy. The script +re-verifies that itself and refuses to report a pass if the repo is importable. + +Prints one JSON object on stdout. Exit code is always 0 — the caller reads `checks` +and decides. (Exiting nonzero here would be indistinguishable from an interpreter crash.) + +Two distinct failure modes are separated on purpose, because they are opposite bugs: + + py-import-module the documented import must WORK from the published artifact + py-no-stdlib-shadow the distribution must not claim a top-level name that collides + with a CPython stdlib module. A collision is a latent disaster in + both directions: on a path where the dist wins, every consumer of + the stdlib module silently gets the wrong library; on a path where + the stdlib wins (the usual one, since the stdlib dir precedes + site-packages), the SDK itself becomes unreachable. +""" +from __future__ import annotations + +import argparse +import importlib +import json +import os +import sys +import sysconfig + + +def realpath(p: str) -> str: + return os.path.realpath(p) if p else "" + + +def check(name: str, ok: bool, detail: str) -> dict: + return {"name": name, "ok": bool(ok), "detail": detail} + + +def dist_top_level(dist_name: str) -> list[str]: + """Top-level import names the installed distribution claims. + + top_level.txt is the cheap answer but is absent from many modern wheels, so fall back + to deriving the set from RECORD. Returns [] only when the distribution is missing. + """ + import importlib.metadata as md + + try: + dist = md.distribution(dist_name) + except md.PackageNotFoundError: + return [] + + raw = None + try: + raw = dist.read_text("top_level.txt") + except Exception: + raw = None + if raw: + return sorted({line.strip() for line in raw.splitlines() if line.strip()}) + + tops: set[str] = set() + for f in dist.files or []: + parts = str(f).split("/") + if not parts or parts[0].endswith(".dist-info") or parts[0].endswith(".data"): + continue + if len(parts) > 1: + tops.add(parts[0]) + elif parts[0].endswith(".py"): + tops.add(parts[0][:-3]) + return sorted(tops) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--dist", required=True, help="installed distribution name, e.g. wave-sdk") + ap.add_argument("--module", required=True, help="documented import name, e.g. wave_sdk") + ap.add_argument("--symbol", default=None, help="symbol the import must expose, e.g. Wave") + args = ap.parse_args() + + stdlib_dir = realpath(sysconfig.get_paths()["stdlib"]) + site_dirs = [realpath(p) for p in sys.path if "site-packages" in p] + checks: list[dict] = [] + + # Guard: a repo checkout on sys.path would make this whole run meaningless. + repo_marker_on_path = [ + p for p in sys.path + if p and os.path.isdir(os.path.join(p, "sdk-python", "wave")) + ] + checks.append(check( + "cleanroom-isolation", + not repo_marker_on_path, + "no repo checkout on sys.path" if not repo_marker_on_path + else f"REPO ON sys.path: {repo_marker_on_path} — results would be untrustworthy", + )) + + # ---- py-import-module ------------------------------------------------------------- + try: + mod = importlib.import_module(args.module) + where = realpath(getattr(mod, "__file__", "") or "") + from_site = any(where.startswith(s + os.sep) for s in site_dirs) + if args.symbol and not hasattr(mod, args.symbol): + checks.append(check( + "py-import-module", False, + f"`import {args.module}` succeeded but has no attribute " + f"`{args.symbol}` (resolved to {where or ''})", + )) + elif not from_site: + checks.append(check( + "py-import-module", False, + f"`{args.module}` resolved to {where}, which is NOT in site-packages — " + f"the published artifact did not provide it", + )) + else: + sym = f".{args.symbol}" if args.symbol else "" + checks.append(check( + "py-import-module", True, + f"`from {args.module} import {args.symbol}` OK" if args.symbol + else f"`import {args.module}`{sym} OK -> {where}", + )) + except Exception as e: # ImportError and anything the package raises at import time + checks.append(check( + "py-import-module", False, + f"`import {args.module}` raised {type(e).__name__}: {str(e)[:200]}", + )) + + # ---- py-no-stdlib-shadow ---------------------------------------------------------- + tops = dist_top_level(args.dist) + stdlib_names = set(getattr(sys, "stdlib_module_names", set())) + collisions = sorted(t for t in tops if t in stdlib_names) + if not tops: + checks.append(check( + "py-no-stdlib-shadow", False, + f"distribution `{args.dist}` is not installed — cannot determine top-level names", + )) + elif collisions: + detail_parts = [] + for c in collisions: + try: + m = importlib.import_module(c) + w = realpath(getattr(m, "__file__", "") or "") + winner = "stdlib" if w.startswith(stdlib_dir + os.sep) else "the distribution" + detail_parts.append(f"`{c}` (import resolves to {winner}: {w})") + except Exception: + detail_parts.append(f"`{c}` (unimportable)") + checks.append(check( + "py-no-stdlib-shadow", False, + f"distribution `{args.dist}` claims top-level name(s) that collide with the " + f"CPython stdlib: " + "; ".join(detail_parts), + )) + else: + checks.append(check( + "py-no-stdlib-shadow", True, + f"top-level names {tops} do not collide with the stdlib", + )) + + print(json.dumps({ + "dist": args.dist, + "module": args.module, + "python": sys.version.split()[0], + "top_level": tops, + "stdlib_dir": stdlib_dir, + "site_packages": site_dirs, + "checks": checks, + })) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ga/mcp-stdio-probe.mjs b/scripts/ga/mcp-stdio-probe.mjs new file mode 100644 index 0000000..f73ba45 --- /dev/null +++ b/scripts/ga/mcp-stdio-probe.mjs @@ -0,0 +1,124 @@ +#!/usr/bin/env node +// Minimal MCP stdio client — initialize -> notifications/initialized -> tools/list. +// +// Deliberately speaks raw JSON-RPC over stdio instead of importing @modelcontextprotocol/sdk: +// the probe must be able to contradict the artifact under test, so it shares no code with it. +// Prints a single JSON object on stdout and exits 0; on failure prints {"ok":false,...} and +// exits 1. The caller (registry-cleanroom.mjs) parses that object. +// +// Usage: node mcp-stdio-probe.mjs + +import { spawn } from 'node:child_process'; + +const RPC_TIMEOUT_MS = Number(process.env.MCP_PROBE_TIMEOUT_MS || 30000); +const entry = process.argv[2]; + +function emit(obj) { + process.stdout.write(JSON.stringify(obj) + '\n'); +} + +if (!entry) { + emit({ ok: false, error: 'usage: mcp-stdio-probe.mjs ' }); + process.exit(1); +} + +const child = spawn(process.execPath, [entry], { + stdio: ['pipe', 'pipe', 'pipe'], + env: { + ...process.env, + // A syntactically plausible but non-functional key. tools/list must not require a live + // credential — if it does, that is itself a defect worth failing on. No real secret is + // used and no authenticated request is expected to succeed. + WAVE_API_KEY: process.env.WAVE_CLEANROOM_API_KEY || 'wave_cleanroom_not_a_real_key', + }, +}); + +let stdoutBuf = ''; +let stderrBuf = ''; +const pending = new Map(); + +child.stderr.on('data', (d) => { + stderrBuf += d.toString(); + if (stderrBuf.length > 8000) stderrBuf = stderrBuf.slice(-8000); +}); + +child.stdout.on('data', (d) => { + stdoutBuf += d.toString(); + let idx; + while ((idx = stdoutBuf.indexOf('\n')) >= 0) { + const line = stdoutBuf.slice(0, idx).trim(); + stdoutBuf = stdoutBuf.slice(idx + 1); + if (!line) continue; + let msg; + try { + msg = JSON.parse(line); + } catch { + continue; // servers sometimes log non-JSON to stdout; ignore rather than crash + } + const resolve = msg && msg.id != null ? pending.get(msg.id) : undefined; + if (resolve) { + pending.delete(msg.id); + resolve(msg); + } + } +}); + +let childExit = null; +child.on('exit', (code, signal) => { + childExit = { code, signal }; + for (const [, resolve] of pending) resolve({ error: { message: `server exited (code=${code} signal=${signal})` } }); + pending.clear(); +}); + +let nextId = 1; +function rpc(method, params) { + const id = nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`timeout after ${RPC_TIMEOUT_MS}ms waiting for ${method}`)); + }, RPC_TIMEOUT_MS); + pending.set(id, (m) => { + clearTimeout(timer); + resolve(m); + }); + child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n'); + }); +} + +function notify(method, params) { + child.stdin.write(JSON.stringify({ jsonrpc: '2.0', method, params }) + '\n'); +} + +function fail(message) { + emit({ ok: false, error: message, stderr: stderrBuf.slice(0, 2000), exit: childExit }); + try { child.kill('SIGKILL'); } catch { /* already gone */ } + process.exit(1); +} + +try { + const init = await rpc('initialize', { + protocolVersion: process.env.MCP_PROTOCOL_VERSION || '2025-06-18', + capabilities: {}, + clientInfo: { name: 'wave-ga-cleanroom', version: '1.0.0' }, + }); + if (init.error) fail('initialize failed: ' + JSON.stringify(init.error).slice(0, 300)); + + notify('notifications/initialized', {}); + + const list = await rpc('tools/list', {}); + if (list.error) fail('tools/list failed: ' + JSON.stringify(list.error).slice(0, 300)); + + const tools = (list.result && list.result.tools) || []; + emit({ + ok: true, + serverInfo: (init.result && init.result.serverInfo) || null, + protocolVersion: (init.result && init.result.protocolVersion) || null, + toolCount: tools.length, + toolNames: tools.map((t) => t.name).sort(), + }); + try { child.kill('SIGKILL'); } catch { /* already gone */ } + process.exit(0); +} catch (err) { + fail(String((err && err.message) || err)); +} diff --git a/scripts/ga/registry-cleanroom.mjs b/scripts/ga/registry-cleanroom.mjs new file mode 100644 index 0000000..1e9375b --- /dev/null +++ b/scripts/ga/registry-cleanroom.mjs @@ -0,0 +1,196 @@ +#!/usr/bin/env node +/** + * registry-cleanroom — GA acceptance for what the PUBLIC REGISTRIES actually serve. + * + * WHY THIS EXISTS + * --------------- + * Every artifact regression in the pre-GA audit reached users while CI was green, because CI was + * testing the repository and the registry was serving something else. A green source branch cannot + * certify a package already published to npm or PyPI. This suite therefore never reads the + * checkout: it installs from the public registry into a throwaway directory or venv, with no repo + * on any module path, no `npm link`, no `pip install -e`, and a freshly generated npm user-config + * so a developer's scoped-registry override or auth token cannot leak in. + * + * It is built to FAIL when the registry is broken. A green run against a broken registry is the one + * outcome worse than having no suite at all. + * + * OUTPUT + * /cleanroom-report.json full detail: resolved versions, digests, per-check results + * /ga-evidence.json WAVE-GA-gate-spec-v1.0.0 evidence fragment, keyed by criterion + * + * EXIT CODES + * 0 every check of every selected target passed + * 1 the gate ran and something failed (a real artifact defect) + * 2 the gate could not run (never to be read as a pass) + * + * USAGE + * node scripts/ga/registry-cleanroom.mjs [--out-dir DIR] [--only id,id] [--python EXE] + * [--versions '@wave-av/cli=1.0.9,wave-sdk=2.1.0'] + * `--versions` (or CLEANROOM_VERSIONS) pins the exact published version to accept, which is how + * the release job tests the versions it just published instead of whatever `latest` points at. + */ + +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { runNpmTarget, runPypiTarget } from './cleanroom-targets.mjs'; +import { bad, run } from './cleanroom-util.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, '..', '..'); + +function parseVersionPins(sink, raw) { + for (const pair of String(raw).split(',')) { + const eq = pair.lastIndexOf('='); + const key = eq > 0 ? pair.slice(0, eq).trim() : ''; + if (key && !(key in sink)) sink[key] = pair.slice(eq + 1).trim(); + } +} + +function parseArgs(argv) { + const out = { + outDir: join(REPO_ROOT, 'ga-out'), + only: null, + python: process.env.CLEANROOM_PYTHON || 'python3', + versions: {}, + }; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + if (a === '--out-dir') out.outDir = resolve(argv[++i]); + else if (a === '--only') out.only = new Set(argv[++i].split(',').map((s) => s.trim()).filter(Boolean)); + else if (a === '--python') out.python = argv[++i]; + else if (a === '--versions') parseVersionPins(out.versions, argv[++i]); + else throw new Error(`unknown argument: ${a}`); + } + if (process.env.CLEANROOM_VERSIONS) parseVersionPins(out.versions, process.env.CLEANROOM_VERSIONS); + return out; +} + +/** + * Fold per-check results into per-criterion evidence. Structural checks (install/download/venv) + * carry no explicit mapping and belong to ART-001: an artifact that will not install has failed + * "published artifacts install" by definition. + */ +function buildEvidence(spec, results, revision) { + const map = spec.criteria_map || {}; + const byCriterion = new Map(); + for (const r of results) { + for (const c of r.checks) { + for (const id of map[c.name] || ['ART-001']) { + if (!byCriterion.has(id)) byCriterion.set(id, []); + byCriterion.get(id).push({ package: `${r.package}@${r.version}`, check: c.name, ok: c.ok, detail: c.detail }); + } + } + } + + // Fingerprint deliberately excludes timestamps, temp paths and durations, per the gate spec's + // idempotency rules: two runs observing the same artifacts must produce the same digest. + const fingerprint = createHash('sha256').update(JSON.stringify( + results + .map((r) => ({ + id: r.id, + package: r.package, + version: r.version, + digest: r.integrity || r.artifact?.sha256 || null, + checks: r.checks.map((c) => [c.name, c.ok]).sort(), + })) + .sort((a, b) => a.id.localeCompare(b.id)), + )).digest('hex'); + + const verifiedAt = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); + return { + fingerprint, + verifiedAt, + evidence: { + spec_version: spec.spec_version || '1.0.0', + repository: spec.repository || 'wave-av/sdks', + revision, + results: [...byCriterion.keys()].sort().map((id) => { + const rows = byCriterion.get(id); + const failures = rows.filter((r) => !r.ok); + return { + criterion_id: id, + status: failures.length === 0 ? 'pass' : 'fail', + command: 'node scripts/ga/registry-cleanroom.mjs', + evidence_sha256: fingerprint, + evidence_uri: 'ci://wave-av/sdks/.github/workflows/registry-cleanroom.yml#cleanroom-report.json', + verified_at: verifiedAt, + targets_observed: [...new Set(rows.map((r) => r.package))].sort(), + failing_checks: failures.map((f) => `${f.package}: ${f.check} — ${f.detail}`), + }; + }), + }, + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const spec = JSON.parse(readFileSync(join(HERE, 'cleanroom-targets.json'), 'utf8')); + const targets = spec.targets.filter((t) => !args.only || args.only.has(t.id)); + if (targets.length === 0) { + throw new Error(`--only matched no targets (known: ${spec.targets.map((t) => t.id).join(', ')})`); + } + + const revision = (run('git', ['rev-parse', 'HEAD'], { cwd: REPO_ROOT }).stdout || '').trim() + || process.env.GITHUB_SHA || 'unknown'; + const startedAt = new Date().toISOString(); + + process.stdout.write(`registry clean-room acceptance — ${targets.length} target(s), repo revision ${revision.slice(0, 12)}\n`); + process.stdout.write('installing ONLY from public registries; no checkout on any module path\n\n'); + + const results = []; + for (const t of targets) { + process.stdout.write(`-- ${t.id} (${t.ecosystem}: ${t.package})\n`); + let r; + try { + r = t.ecosystem === 'npm' ? await runNpmTarget(t, args) : await runPypiTarget(t, args); + } catch (e) { + r = { + id: t.id, ecosystem: t.ecosystem, package: t.package, version: 'unresolved', + checks: [bad('resolve', `${e?.name}: ${String(e?.message).slice(0, 300)}`)], + }; + } + for (const c of r.checks) process.stdout.write(` ${c.ok ? 'PASS' : 'FAIL'} ${c.name}: ${c.detail}\n`); + process.stdout.write('\n'); + results.push(r); + } + + const { evidence, fingerprint, verifiedAt } = buildEvidence(spec, results, revision); + mkdirSync(args.outDir, { recursive: true }); + writeFileSync(join(args.outDir, 'cleanroom-report.json'), `${JSON.stringify({ + schema: 'wave-registry-cleanroom/1', + spec_version: spec.spec_version || '1.0.0', + repository: spec.repository || 'wave-av/sdks', + revision, + started_at: startedAt, + finished_at: verifiedAt, + evidence_sha256: fingerprint, + runner: { node: process.version, platform: process.platform, python: args.python }, + version_pins: args.versions, + targets: results, + }, null, 2)}\n`); + writeFileSync(join(args.outDir, 'ga-evidence.json'), `${JSON.stringify(evidence, null, 2)}\n`); + + process.stdout.write(`${'-'.repeat(78)}\n`); + for (const row of evidence.results) { + process.stdout.write(`${row.criterion_id}: ${row.status.toUpperCase()} (${row.targets_observed.length} artifacts observed)\n`); + } + process.stdout.write(`\nevidence fingerprint: ${fingerprint}\n`); + process.stdout.write(`wrote ${join(args.outDir, 'cleanroom-report.json')} and ${join(args.outDir, 'ga-evidence.json')}\n`); + + const failures = results.flatMap((r) => r.checks.filter((c) => !c.ok).map((c) => `${r.package}@${r.version} ${c.name}`)); + if (failures.length > 0) { + process.stdout.write(`\nREGISTRY CLEAN-ROOM FAILED: ${failures.length} check(s)\n`); + for (const f of failures) process.stdout.write(` - ${f}\n`); + process.exitCode = 1; + return; + } + process.stdout.write('\nregistry clean-room: all checks passed\n'); +} + +main().catch((e) => { + process.stderr.write(`registry-cleanroom could not run: ${e?.stack || e}\n`); + process.exit(2); // distinct from 1 so CI can tell "gate failed" from "gate never ran" +}); From 2028399f7ded45ed6cef75f9769c64a0ba2b26e4 Mon Sep 17 00:00:00 2001 From: Jake Fineman Date: Thu, 3 Sep 2026 21:28:55 -0400 Subject: [PATCH 2/2] fix(ga): encode npm registry path segments with encodeURIComponent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL js/incomplete-sanitization (high, alert #1) on scripts/ga/cleanroom-util.mjs:41. `npmEncode` escaped the package name with `pkg.replace("/", "%2f")`. A string first argument to `replace` substitutes only the FIRST match, so every separator after the first survived into the registry URL as a live path separator: "a/../../x".replace("/", "%2f") === "a%2f../../x" The fetch then resolves against a different registry endpoint than the caller asked for. The values are not all repo-controlled: `--versions` pins reach this from the workflow_dispatch `versions` input via CLEANROOM_VERSIONS, so a dispatcher-supplied string could steer the gate onto an endpoint describing an artifact nobody installs — a clean-room suite that can be pointed away from the artifact under test is not a control. Fixed with encodeURIComponent, the platform primitive CodeQL recommends over a hand-rolled escaper. It escapes every occurrence and every other URL meta-character, and is already the encoding the PyPI path in this same file uses for both name and version. Also encoded the `version` half of the packument URL, which was interpolated with no escaping at all. No suppression comment, no weakened assertion. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/ga/cleanroom-targets.mjs | 5 ++++- scripts/ga/cleanroom-util.mjs | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/scripts/ga/cleanroom-targets.mjs b/scripts/ga/cleanroom-targets.mjs index 4d1f07b..42b0698 100644 --- a/scripts/ga/cleanroom-targets.mjs +++ b/scripts/ga/cleanroom-targets.mjs @@ -18,7 +18,10 @@ export async function runNpmTarget(target, args) { const pkg = target.package; const latest = await fetchJson(`${PUBLIC_NPM}/${npmEncode(pkg)}/latest`); const version = args.versions[pkg] || latest.version; - const packument = version === latest.version ? latest : await fetchJson(`${PUBLIC_NPM}/${npmEncode(pkg)}/${version}`); + // `version` is encoded for the same reason the package name is: it can come from the + // `workflow_dispatch` versions input, so it is not repo-controlled and must not be able to + // introduce a path separator. The PyPI path below already encodes both halves. + const packument = version === latest.version ? latest : await fetchJson(`${PUBLIC_NPM}/${npmEncode(pkg)}/${npmEncode(version)}`); const result = { id: target.id, diff --git a/scripts/ga/cleanroom-util.mjs b/scripts/ga/cleanroom-util.mjs index 473ede7..97fedbb 100644 --- a/scripts/ga/cleanroom-util.mjs +++ b/scripts/ga/cleanroom-util.mjs @@ -38,7 +38,23 @@ export async function fetchJson(url) { export function ok(name, detail) { return { name, ok: true, detail }; } export function bad(name, detail) { return { name, ok: false, detail }; } -export function npmEncode(pkg) { return pkg.replace('/', '%2f'); } +/** + * Encode one value as a SINGLE npm-registry URL path segment. + * + * `encodeURIComponent`, not a hand-rolled `replace`. `pkg.replace('/', '%2f')` — the previous + * implementation — escapes only the FIRST occurrence, because a string (rather than a global regex) + * first argument replaces once. Every later separator survives into the URL as a real path + * separator: `'a/../../x'.replace('/', '%2f')` is `'a%2f../../x'`, so the fetch resolves against a + * different registry endpoint than the caller asked for. That matters here because the values are + * not all repo-controlled — `--versions` pins arrive from the `workflow_dispatch` input via + * `CLEANROOM_VERSIONS` — and because a gate that can be steered onto the wrong endpoint is a gate + * that can be made to report on an artifact nobody installs. + * + * The platform primitive escapes every occurrence and every other URL meta-character, and is the + * encoding the PyPI path in this suite already uses. It is the identity function for ordinary + * semver, including prereleases. + */ +export function npmEncode(value) { return encodeURIComponent(value); } /** Install one package from the PUBLIC npm registry into a throwaway directory. */ export function npmCleanRoom(pkgName, version) {