diff --git a/.github/workflows/registry-cleanroom.yml b/.github/workflows/registry-cleanroom.yml new file mode 100644 index 0000000..e06b0f8 --- /dev/null +++ b/.github/workflows/registry-cleanroom.yml @@ -0,0 +1,155 @@ +name: registry clean-room acceptance + +# Tests ONLY what the public registries serve — never this checkout. +# +# `registry parity` (the sibling workflow) answers "does the declared version equal the published +# version". That is necessary and not sufficient: it never installs anything, so it cannot see a +# package whose version number is right and whose CONTENTS are broken. Every artifact regression in +# the pre-GA audit was of that second kind — CI green on source while npm and PyPI served a broken +# build for days. This workflow installs the published artifact into a throwaway directory or venv +# and asserts it actually works. +# +# WHY NIGHTLY AND NOT ONLY ON RELEASE +# A published package can break with NO commit anywhere. @wave-av/cli's own published dependency +# range on @wave-av/sdk is a caret range, so what a customer receives is decided by npm's resolver +# on the day they install, not by anything in this repo. Only a scheduled run catches that. +# +# BLOCKING BEHAVIOUR +# schedule / workflow_run(release) / workflow_dispatch -> HARD FAIL, and open-or-update an issue. +# pull_request -> informational only. +# A PR is not the cause of an already-published defect, so a PR is not blocked by one. The PR run +# exists so that a change to the gate itself is exercised before it merges. There is deliberately +# NO path filter: the check reports on every PR, which is what makes it eligible to become a +# required status check later (a path-filtered required check stays permanently unreported and +# blocks every PR that misses the filter — the lesson already recorded in registry-parity.yml). + +on: + schedule: + # 09:00 UTC — deliberately offset from `registry parity` (14:00) so a registry outage does not + # take out both signals in the same window. + - cron: "0 9 * * *" + workflow_dispatch: + inputs: + versions: + description: 'Exact versions to accept, e.g. "@wave-av/cli=1.0.9,wave-sdk=2.1.0" (default: registry latest)' + type: string + required: false + only: + description: 'Comma-separated target ids to run (default: all). Ids are in scripts/ga/cleanroom-targets.json' + type: string + required: false + workflow_run: + # Runs after a real publish so a release is verified against the registry it just wrote to. + workflows: ["npm publish (OIDC + provenance)"] + types: [completed] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + cleanroom: + runs-on: ubuntu-latest + timeout-minutes: 30 + # On workflow_run, only a SUCCESSFUL publish is worth verifying; a failed publish has its own + # error and would produce a confusing second failure here. + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + permissions: + contents: read + issues: write # fail-loud: open or update the tracking issue + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Run clean-room acceptance against the public registries + id: cleanroom + env: + CLEANROOM_VERSIONS: ${{ inputs.versions }} + ONLY: ${{ inputs.only }} + run: | + set -uo pipefail + args=(--python python3 --out-dir "$GITHUB_WORKSPACE/ga-out") + [ -n "${ONLY:-}" ] && args+=(--only "$ONLY") + set +e + node scripts/ga/registry-cleanroom.mjs "${args[@]}" 2>&1 | tee "$RUNNER_TEMP/cleanroom.log" + code=${PIPESTATUS[0]} + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + { + echo "## Registry clean-room acceptance" + echo + echo "Exit code \`$code\` (0 = all checks passed, 1 = an artifact failed, 2 = the gate could not run)." + echo + echo '```' + cat "$RUNNER_TEMP/cleanroom.log" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Upload GA evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ga-evidence-registry-cleanroom + path: ga-out/ + if-no-files-found: warn + retention-days: 90 + + - name: Fail loudly — open or update the tracking issue + # A nightly that fails quietly is worse than no nightly at all. + if: steps.cleanroom.outputs.exit_code != '0' && github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + EXIT_CODE: ${{ steps.cleanroom.outputs.exit_code }} + run: | + set -euo pipefail + title="registry clean-room acceptance is failing" + # Only the failing-check lines: the issue must say WHAT is broken, not page a human to + # go read a log to find out. + summary=$(sed -n '/^REGISTRY CLEAN-ROOM FAILED/,$p' "$RUNNER_TEMP/cleanroom.log" | head -40) + [ -z "$summary" ] && summary="The gate did not complete (exit $EXIT_CODE). See the run log." + body=$(printf '%s\n\n```\n%s\n```\n\nRun: %s\nEvent: %s\n\nThe published artifacts do not satisfy ART-001 / SUPPLY-001 / VER-001. A green source branch does not certify these — the registry is what customers install.\n' \ + "Clean-room acceptance against the public registries failed." "$summary" "$RUN_URL" "$GITHUB_EVENT_NAME") + existing=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "$title in:title" --json number --jq '.[0].number // empty') + if [ -n "$existing" ]; then + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" --body "$body" + echo "updated issue #$existing" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body" \ + || echo "::warning::could not open the tracking issue; the job still fails below" + fi + + - name: Enforce + # Hard-fail everywhere the result is actionable. A pull request is informational: it did not + # publish the artifact under test and cannot fix it. + if: github.event_name != 'pull_request' + env: + CODE: ${{ steps.cleanroom.outputs.exit_code }} + run: | + if [ "$CODE" != "0" ]; then + echo "::error title=registry clean-room::published artifacts failed clean-room acceptance (exit $CODE) — see the job summary" + exit 1 + fi + echo "registry clean-room acceptance passed" + + - name: Report (pull request, informational) + if: github.event_name == 'pull_request' + env: + CODE: ${{ steps.cleanroom.outputs.exit_code }} + run: | + [ "$CODE" = "0" ] && echo "clean-room acceptance passed" \ + || echo "::warning title=registry clean-room::published artifacts fail clean-room acceptance (exit $CODE). Not blocking this PR — see the job summary." diff --git a/GA-READINESS.md b/GA-READINESS.md new file mode 100644 index 0000000..2b4b047 --- /dev/null +++ b/GA-READINESS.md @@ -0,0 +1,164 @@ +# WAVE GA compliance — wave-av/sdks + +> Criterion IDs are global and immutable. `unknown` and a waiver are **not** a pass. A green +> repository is necessary but not sufficient for platform GA. + +## Repository declaration + +```yaml +repository: wave-av/sdks +revision: 70b2a04a205658a25b3723adec9faf665e204af6 +repo_class: sdk +owner: WAVE platform / SDK publishing +last_evaluated_at: 2026-09-04T00:31:56Z +evaluator_version: scripts/ga/registry-cleanroom.mjs +spec_version: 1.0.0 +overall_status: fail +``` + +`overall_status: fail` is the honest reading, and it is an improvement on what came before it: +these criteria were **unknown** until this repository could install what the registries serve and +look. The gate now produces evidence. The evidence says the published artifacts are broken. + +## How this repo proves compliance + +`scripts/ga/registry-cleanroom.mjs` installs each published artifact from its **public registry** +into a throwaway directory or venv — never from this checkout, never `npm link`, never +`pip install -e` — and asserts it behaves. It emits `ga-out/ga-evidence.json` keyed to criterion +IDs, plus `ga-out/cleanroom-report.json` with per-check detail and artifact digests. The run +output is a CI artifact and is never committed; a committed report would let a stale file +masquerade as current evidence. + +The distinction from the sibling `registry parity` workflow matters. Parity asks *"does the +declared version equal the published version"* and never installs anything, so it cannot see a +package whose version number is correct and whose contents are broken. Every artifact regression in +the pre-GA audit was of that second kind. + +```bash +# everything the registries currently serve +node scripts/ga/registry-cleanroom.mjs + +# a specific release, pinned rather than whatever `latest` points at +node scripts/ga/registry-cleanroom.mjs --versions '@wave-av/cli=1.0.9,wave-sdk=2.1.0' +``` + +Exit `0` all checks passed · `1` an artifact failed · `2` the gate could not run. Exit 2 is never +to be read as a pass. + +Schedule: nightly at 09:00 UTC, after every successful npm publish, on demand, and informationally +on every pull request (`.github/workflows/registry-cleanroom.yml`). Nightly is not decoration — a +published package can break with no commit anywhere, because a published dependency **range** is +resolved on the day a customer installs. + +| Criterion | Title | GA must-pass | Repo status | Evidence | +|---|---|---:|---|---| +| ART-001 | Published artifacts install, import, start, identify themselves, and match source | true | **fail** | `ga-out/ga-evidence.json` (evaluator: `scripts/ga/registry-cleanroom.mjs`) | +| SUPPLY-001 | Builds have provenance, signatures, SBOMs, dependency policy, and protected release identity | true | **fail** (partial coverage — see note) | `ga-out/ga-evidence.json` | +| VER-001 | Version and release truth agree from source through deployment | true | **fail** (registry half) | `ga-out/ga-evidence.json` | +| CONTRACT-001 | One promoted contract is the source of truth across spec, gateway, registry, MCP, SDK and CLI | true | unknown | not evaluated by this repo | +| COMPAT-001 | Backward compatibility, versioning, deprecation, and sunset policy are enforced | true | unknown | not evaluated by this repo | +| DX-001 | A new developer can complete one honest golden path from published materials | true | unknown | not evaluated by this repo | +| STATUS-001 | Marketing, registry, preview labels, availability, and status tell the same truth | true | unknown | not evaluated by this repo | + +Criteria absent from this table are owned by other repositories and surfaces; this repository makes +no claim about them. Absence here is `unknown` at the platform gate, not `not_applicable`. + +## Per-criterion evidence + +```yaml +criterion_id: ART-001 +status: fail +owner: WAVE platform / SDK publishing +verification_command: node scripts/ga/registry-cleanroom.mjs +verified_revision: 70b2a04a205658a25b3723adec9faf665e204af6 +verified_at: 2026-09-04T00:31:56Z +evidence: + - uri: ci://wave-av/sdks/.github/workflows/registry-cleanroom.yml#cleanroom-report.json + sha256: 0f03603df2bc33f50958a75c8998777463507c1f9e3ee0fec811d82a3ca1206d +pass_condition_from_spec: > + All supported runtimes pass from public registries; Python uses a non-stdlib-colliding import; + source, package metadata, CLI banner, tag and GitHub release agree. +notes: | + Observed against the live registries on 2026-09-04. Passing: + @wave-av/sdk@2.1.3 ESM import, CJS require, all 46 declared subpath exports resolve. + @wave-av/adk@1.0.15 installs and imports. + @wave-av/mcp-server@0.2.0 starts over stdio, lists 18 tools, serves every tool its shipped + README advertises. + Failing: + @wave-av/cli@1.0.8 `wave --version` prints 1.0.0. Installs cleanly and `--help` exits 0, + so nothing short of running the binary detects this. + wave-sdk@2.0.0 (PyPI) `from wave_sdk import Wave` raises ModuleNotFoundError. The wheel's + only top-level name is `wave`, which collides with the CPython stdlib + module of that name; because the stdlib directory precedes + site-packages, `import wave` returns the stdlib WAV reader and the SDK + is unreachable by any name. The artifact is unusable as published. + wave-av-sdk@2.0.0 (PyPI) identical defect. +``` + +```yaml +criterion_id: VER-001 +status: fail +owner: WAVE platform / SDK publishing +verification_command: node scripts/ga/registry-cleanroom.mjs +verified_revision: 70b2a04a205658a25b3723adec9faf665e204af6 +verified_at: 2026-09-04T00:31:56Z +evidence: + - uri: ci://wave-av/sdks/.github/workflows/registry-cleanroom.yml#cleanroom-report.json + sha256: 0f03603df2bc33f50958a75c8998777463507c1f9e3ee0fec811d82a3ca1206d +pass_condition_from_spec: > + Every shipped component resolves to one source revision and version; no newer source is + represented as deployed; mutable channels are labeled; deployment receipt identifies artifact + digest. +notes: | + This repository covers the REGISTRY half of VER-001 — does the artifact agree with itself about + which build it is. Two published artifacts do not: + @wave-av/cli@1.0.8 binary self-reports 1.0.0 + @wave-av/mcp-server@0.2.0 serverInfo.version reports 0.1.0 + In both cases the package metadata is correct and the code inside it disagrees, so a version + comparison against the registry cannot see the defect — only running the artifact can. + Tag/GitHub-release/deployed-endpoint agreement is NOT covered here and remains unknown; it belongs + to the release-ledger check named in the spec's runnable_command. +``` + +```yaml +criterion_id: SUPPLY-001 +status: fail +owner: WAVE platform / SDK publishing +verification_command: node scripts/ga/registry-cleanroom.mjs +verified_revision: 70b2a04a205658a25b3723adec9faf665e204af6 +verified_at: 2026-09-04T00:31:56Z +evidence: + - uri: ci://wave-av/sdks/.github/workflows/registry-cleanroom.yml#cleanroom-report.json + sha256: 0f03603df2bc33f50958a75c8998777463507c1f9e3ee0fec811d82a3ca1206d +pass_condition_from_spec: > + Release artifacts are built by approved CI from an immutable source revision, provenance is + verifiable, SBOM is attached, critical known vulnerabilities are resolved or explicitly + risk-accepted, and publisher accounts require strong MFA. +notes: | + PARTIAL COVERAGE — this evaluator checks two of the five clauses. A `fail` here is therefore + sound, but a future `pass` would NOT be sufficient to pass SUPPLY-001 on its own. + Covered and failing: + provenance @wave-av/cli@1.0.8 carries no npm provenance attestation (dist.attestations is + null), while sdk, mcp-server and adk each carry a + https://slsa.dev/provenance/v1 attestation. The CLI was published outside the + provenance-emitting pipeline and cannot be traced to an approved CI build. + dependency @wave-av/cli@1.0.8 declares `@wave-av/sdk: "^2.0.11"`. What a customer receives + policy is decided by npm's resolver on the day they install; today that is 2.1.3. The + published artifact is not reproducible, and this is the precise mechanism by + which a broken SDK shipped inside a CLI that no one had changed. + NOT covered here, still unknown: SBOM attachment, vulnerability posture, publisher MFA and + branch-protection attestation. +``` + +## Operator actions to finish these criteria + +1. **Make the release gate blocking.** The clean-room job hard-fails on schedule, release and + dispatch, but it is not yet a *required* status check. Add + `registry clean-room acceptance / cleanroom` to the default branch's required checks. It runs on + every pull request with no path filter precisely so it can be made required without going + permanently unreported. +2. **Fix the three artifact defects the gate found** (each needs a publish, which is a named floor + and not this lane's to cross): the CLI version constant, the MCP server's `serverInfo.version`, + and the Python distribution's top-level module name. +3. **Republish the CLI through the provenance-emitting workflow** so `dist.attestations` is + populated, and exact-pin its first-party dependency. diff --git a/README.md b/README.md index 49c682a..70cf198 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,20 @@ Agents and short-lived sandboxes can skip the SDK entirely and call the gateway sdk-typescript/packages/{core,clips,sdk}/ # the TS fleet (pnpm workspace) scripts/sync-from-monorepo.sh # allowlist sync + fail-closed secret scan scripts/check-registry-parity.py # declared-vs-published gate (vendored from wave-foundation) +scripts/ga/registry-cleanroom.mjs # clean-room acceptance of what the REGISTRIES serve .github/workflows/publish-npm.yml # OIDC trusted publishing + provenance -.github/workflows/registry-parity.yml # parity gate +.github/workflows/registry-parity.yml # parity gate (declared version == published version) +.github/workflows/registry-cleanroom.yml # clean-room gate (published artifact actually works) ``` +## GA readiness + +This repository participates in the WAVE GA gate. Its criterion ownership, runnable checks and latest evidence are in [GA-READINESS.md](./GA-READINESS.md). CI emits `ga-evidence.json` against spec version 1.0.0. Stable criterion IDs are never renamed or reused. + +The check that matters here is **clean-room acceptance**: `scripts/ga/registry-cleanroom.mjs` installs each package from its public registry into a throwaway directory or venv — never from this checkout — and asserts the artifact imports, starts, and reports its own version honestly. A green source branch cannot certify a package already on npm or PyPI, which is why this runs nightly as well as on release: a published dependency *range* is resolved on the day a customer installs, so an artifact can break with no commit anywhere. + +A green repository is necessary but not sufficient for platform GA. The platform release is **NO-GO** unless the root gate aggregator has current passing evidence for every must-pass criterion across all owning repositories and deployed surfaces. `unknown`, stale evidence, missing evidence, a waiver, or an unapproved `not_applicable` counts as failure. + ## License Apache-2.0 © WAVE — see [LICENSE](LICENSE) and [NOTICE](NOTICE). diff --git a/ga-out/.gitignore b/ga-out/.gitignore new file mode 100644 index 0000000..1dc2cff --- /dev/null +++ b/ga-out/.gitignore @@ -0,0 +1,8 @@ +# Clean-room run output (cleanroom-report.json, ga-evidence.json) is a CI artifact, never a +# committed one: evidence must describe a run that actually happened, in the environment that +# happened to run it. Committing a locally generated report would let a stale file masquerade as +# current evidence — the exact failure mode the GA gate exists to prevent. +# +# Self-ignoring so the root .gitignore does not need to change. +* +!.gitignore diff --git a/scripts/ga/cleanroom-checks.mjs b/scripts/ga/cleanroom-checks.mjs new file mode 100644 index 0000000..dac62ea --- /dev/null +++ b/scripts/ga/cleanroom-checks.mjs @@ -0,0 +1,168 @@ +// The acceptance checks themselves. Each takes a per-target context and returns one result. +// +// Design rule: a check reports the artifact's OWN words back (its version strings, its tool list, +// its resolved module paths) rather than asserting a value hardcoded here. A suite that encodes +// the expected answer drifts into agreeing with itself; a suite that quotes the artifact can only +// pass when the artifact is actually coherent. + +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { bad, installedManifest, ok, run } from './cleanroom-util.mjs'; + +const DEFAULT_ADVERTISED_TOOL_PATTERN = '(?:wave|mvp)_[a-z0-9_]+'; + +export const CHECKS = { + 'npm-provenance-attested': async (ctx) => { + const att = ctx.packument?.dist?.attestations; + if (att?.provenance?.predicateType) { + return ok('npm-provenance-attested', `provenance attestation present (${att.provenance.predicateType})`); + } + return bad('npm-provenance-attested', + `${ctx.pkg}@${ctx.version} has NO provenance attestation on npm (dist.attestations=${JSON.stringify(att ?? null)}). ` + + 'The published artifact cannot be traced to an approved CI build of an immutable source revision.'); + }, + + 'esm-import': async (ctx) => { + const f = join(ctx.room, 'probe-esm.mjs'); + writeFileSync(f, `import * as m from ${JSON.stringify(ctx.pkg)};\nconsole.log(Object.keys(m).length);\n`); + const r = run(process.execPath, [f], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + return r.status === 0 + ? ok('esm-import', `static ESM \`import * from '${ctx.pkg}'\` OK (${r.stdout.trim()} named exports)`) + : bad('esm-import', `static ESM import failed: ${(r.stderr || r.stdout).trim().slice(0, 400)}`); + }, + + 'cjs-require': async (ctx) => { + const f = join(ctx.room, 'probe-cjs.cjs'); + writeFileSync(f, `const m = require(${JSON.stringify(ctx.pkg)});\nconsole.log(Object.keys(m).length);\n`); + const r = run(process.execPath, [f], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + return r.status === 0 + ? ok('cjs-require', `\`require('${ctx.pkg}')\` OK (${r.stdout.trim()} keys)`) + : bad('cjs-require', `CJS require failed: ${(r.stderr || r.stdout).trim().slice(0, 400)}`); + }, + + 'subpath-exports': async (ctx) => { + const exportsMap = ctx.manifest?.exports; + if (!exportsMap || typeof exportsMap !== 'object') { + return bad('subpath-exports', `${ctx.pkg}@${ctx.version} declares no "exports" map — subpath resolution is unverifiable`); + } + const subpaths = Object.keys(exportsMap).filter((k) => k !== './package.json'); + const f = join(ctx.room, 'probe-subpaths.mjs'); + writeFileSync(f, [ + `const subpaths = ${JSON.stringify(subpaths)};`, + `const pkg = ${JSON.stringify(ctx.pkg)};`, + 'const failed = [];', + 'for (const sp of subpaths) {', + " const spec = sp === '.' ? pkg : pkg + '/' + sp.replace(/^\\.\\//, '');", + " try { await import(spec); } catch (e) { failed.push(spec + ': ' + (e.code || e.name)); }", + '}', + 'console.log(JSON.stringify({ total: subpaths.length, failed }));', + ].join('\n')); + const r = run(process.execPath, [f], { cwd: ctx.room, env: ctx.env, timeout: 180000 }); + if (r.status !== 0) return bad('subpath-exports', `subpath probe crashed: ${(r.stderr || r.stdout).trim().slice(0, 400)}`); + let parsed; + try { parsed = JSON.parse(r.stdout.trim().split('\n').pop()); } + catch { return bad('subpath-exports', `unparseable probe output: ${r.stdout.slice(0, 200)}`); } + return parsed.failed.length === 0 + ? ok('subpath-exports', `all ${parsed.total} declared subpath exports resolve`) + : bad('subpath-exports', `${parsed.failed.length}/${parsed.total} declared subpath exports fail to resolve: ${parsed.failed.slice(0, 8).join('; ')}`); + }, + + 'bin-help-exit-zero': async (ctx) => { + const bin = ctx.binPath(); + if (!bin) return bad('bin-help-exit-zero', `no bin found for ${ctx.pkg} (declared bin: ${JSON.stringify(ctx.manifest?.bin ?? null)})`); + const r = run(process.execPath, [bin, '--help'], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + return r.status === 0 + ? ok('bin-help-exit-zero', `\`${ctx.target.bin} --help\` exited 0 (${r.stdout.split('\n').length} lines)`) + : bad('bin-help-exit-zero', `\`${ctx.target.bin} --help\` exited ${r.status}: ${(r.stderr || r.stdout).trim().slice(0, 400)}`); + }, + + 'bin-version-matches-package': async (ctx) => { + const bin = ctx.binPath(); + if (!bin) return bad('bin-version-matches-package', `no bin found for ${ctx.pkg}`); + const r = run(process.execPath, [bin, '--version'], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + if (r.status !== 0) return bad('bin-version-matches-package', `\`${ctx.target.bin} --version\` exited ${r.status}: ${(r.stderr || r.stdout).trim().slice(0, 300)}`); + const printed = (r.stdout.trim().match(/\d+\.\d+\.\d+[^\s]*/) || [r.stdout.trim()])[0]; + return printed === ctx.manifest.version + ? ok('bin-version-matches-package', `\`${ctx.target.bin} --version\` prints ${printed}, matching the installed package version`) + : bad('bin-version-matches-package', + `VERSION LIE: npm served ${ctx.pkg}@${ctx.manifest.version} but \`${ctx.target.bin} --version\` prints ${printed}. ` + + 'A user cannot tell which build they are running, and a bug report cannot be tied to a revision.'); + }, + + 'bin-help-banner-version-consistent': async (ctx) => { + const bin = ctx.binPath(); + if (!bin) return bad('bin-help-banner-version-consistent', `no bin found for ${ctx.pkg}`); + const r = run(process.execPath, [bin, '--help'], { cwd: ctx.room, env: ctx.env, timeout: 120000 }); + // Only lines that actually advertise a version count. A bare semver elsewhere in help text + // (an example payload, a protocol number) must not manufacture a false failure. + const claimed = new Set(); + for (const line of `${r.stdout}\n${r.stderr}`.split('\n')) { + if (!/version|\bv\d/i.test(line)) continue; + for (const m of line.matchAll(/\bv?(\d+\.\d+\.\d+)\b/g)) claimed.add(m[1]); + } + if (claimed.size === 0) return ok('bin-help-banner-version-consistent', 'help output advertises no version string (nothing to contradict)'); + const wrong = [...claimed].filter((v) => v !== ctx.manifest.version); + return wrong.length === 0 + ? ok('bin-help-banner-version-consistent', `help banner advertises ${[...claimed].join(', ')}, matching the installed version`) + : bad('bin-help-banner-version-consistent', `help banner advertises version(s) ${wrong.join(', ')} but npm served ${ctx.manifest.version}`); + }, + + 'declared-dep-ranges-pinned': async (ctx) => { + // A floating range on a first-party sibling put a broken SDK inside a "known good" CLI with + // no commit anywhere. A published artifact whose own dependencies can move is not reproducible, + // which is why this must be checked nightly and not only at release. + const firstParty = Object.entries(ctx.manifest?.dependencies || {}).filter(([n]) => n.startsWith('@wave-av/')); + if (firstParty.length === 0) return ok('declared-dep-ranges-pinned', 'no first-party runtime dependencies to pin'); + const resolved = firstParty.map(([n]) => `${n}@${installedManifest(ctx.room, n)?.version ?? ''}`); + const floating = firstParty.filter(([, range]) => !/^\d+\.\d+\.\d+/.test(range)); + return floating.length === 0 + ? ok('declared-dep-ranges-pinned', `first-party deps exact-pinned; resolved to ${resolved.join(', ')}`) + : bad('declared-dep-ranges-pinned', + `${ctx.pkg}@${ctx.manifest.version} declares floating first-party range(s) ` + + `${floating.map(([n, r]) => `${n}: "${r}"`).join(', ')} — today they resolve to ${resolved.join(', ')}, ` + + 'but tomorrow the same published artifact can ship a different sibling with no commit anywhere.'); + }, + + 'mcp-server-lists-tools': async (ctx) => { + const probe = await ctx.mcpProbe(); + if (!probe.ok) return bad('mcp-server-lists-tools', `server did not complete initialize + tools/list: ${probe.error}`); + return probe.toolCount > 0 + ? ok('mcp-server-lists-tools', `server started and listed ${probe.toolCount} tools: ${probe.toolNames.slice(0, 6).join(', ')}${probe.toolCount > 6 ? ', …' : ''}`) + : bad('mcp-server-lists-tools', 'server started but advertises zero tools'); + }, + + 'mcp-serverinfo-version-matches-package': async (ctx) => { + const probe = await ctx.mcpProbe(); + if (!probe.ok) return bad('mcp-serverinfo-version-matches-package', `could not reach serverInfo: ${probe.error}`); + const reported = probe.serverInfo?.version ?? null; + return reported === ctx.manifest.version + ? ok('mcp-serverinfo-version-matches-package', `serverInfo.version ${reported} matches the installed package version`) + : bad('mcp-serverinfo-version-matches-package', + `VERSION LIE: npm served ${ctx.pkg}@${ctx.manifest.version} but the running server reports ` + + `serverInfo.version=${JSON.stringify(reported)}. An MCP client cannot identify the build it connected to.`); + }, + + 'mcp-advertised-tools-are-served': async (ctx) => { + const readmePath = join(ctx.room, 'node_modules', ...ctx.pkg.split('/'), 'README.md'); + if (!existsSync(readmePath)) { + return bad('mcp-advertised-tools-are-served', `${ctx.pkg}@${ctx.version} ships no README.md — its advertised tool surface cannot be verified against what it serves`); + } + const probe = await ctx.mcpProbe(); + if (!probe.ok) return bad('mcp-advertised-tools-are-served', `could not list served tools: ${probe.error}`); + // Only backticked identifiers count. Bare prose matching picks up things like the API-key + // example `wave_live_...` and would fabricate a failure. + const pattern = new RegExp('`(' + (ctx.target.advertised_tool_pattern || DEFAULT_ADVERTISED_TOOL_PATTERN) + ')`', 'g'); + const advertised = [...new Set([...readFileSync(readmePath, 'utf8').matchAll(pattern)].map((m) => m[1]))].sort(); + if (advertised.length === 0) return bad('mcp-advertised-tools-are-served', 'shipped README advertises no tool names — the artifact documents nothing it serves'); + const served = new Set(probe.toolNames); + const missing = advertised.filter((t) => !served.has(t)); + if (missing.length > 0) { + return bad('mcp-advertised-tools-are-served', `shipped README advertises ${missing.length} tool(s) the server does not serve: ${missing.join(', ')}`); + } + const undocumented = probe.toolNames.filter((t) => !advertised.includes(t)); + const extra = undocumented.length + ? ` (${undocumented.length} served but undocumented: ${undocumented.slice(0, 5).join(', ')}${undocumented.length > 5 ? ', …' : ''})` + : ''; + return ok('mcp-advertised-tools-are-served', `all ${advertised.length} README-advertised tools are served${extra}`); + }, +}; diff --git a/scripts/ga/cleanroom-targets.json b/scripts/ga/cleanroom-targets.json new file mode 100644 index 0000000..e1f99c2 --- /dev/null +++ b/scripts/ga/cleanroom-targets.json @@ -0,0 +1,96 @@ +{ + "$comment": [ + "Declarative target manifest for scripts/ga/registry-cleanroom.mjs.", + "Every entry describes an artifact THE PUBLIC REGISTRY SERVES — never a local checkout.", + "`checks` names are implemented in registry-cleanroom.mjs (CHECKS table); each check", + "maps to the GA criterion IDs it produces evidence for. Adding a package here is the", + "only edit needed to bring it under the clean-room gate." + ], + "spec_version": "1.0.0", + "repository": "wave-av/sdks", + "targets": [ + { + "id": "npm-sdk", + "ecosystem": "npm", + "package": "@wave-av/sdk", + "checks": [ + "npm-provenance-attested", + "esm-import", + "cjs-require", + "subpath-exports" + ] + }, + { + "id": "npm-cli", + "ecosystem": "npm", + "package": "@wave-av/cli", + "bin": "wave", + "checks": [ + "npm-provenance-attested", + "bin-help-exit-zero", + "bin-version-matches-package", + "bin-help-banner-version-consistent", + "declared-dep-ranges-pinned" + ] + }, + { + "id": "npm-mcp-server", + "ecosystem": "npm", + "package": "@wave-av/mcp-server", + "bin": "wave-mcp-server", + "checks": [ + "npm-provenance-attested", + "esm-import", + "mcp-server-lists-tools", + "mcp-serverinfo-version-matches-package", + "mcp-advertised-tools-are-served" + ] + }, + { + "id": "npm-adk", + "ecosystem": "npm", + "package": "@wave-av/adk", + "checks": [ + "npm-provenance-attested", + "esm-import" + ] + }, + { + "id": "pypi-wave-sdk", + "ecosystem": "pypi", + "package": "wave-sdk", + "import_module": "wave_sdk", + "import_symbol": "Wave", + "checks": [ + "py-import-module", + "py-no-stdlib-shadow" + ] + }, + { + "id": "pypi-wave-av-sdk", + "ecosystem": "pypi", + "package": "wave-av-sdk", + "import_module": "wave_sdk", + "import_symbol": "Wave", + "checks": [ + "py-import-module", + "py-no-stdlib-shadow" + ] + } + ], + "criteria_map": { + "npm-provenance-attested": ["SUPPLY-001"], + "esm-import": ["ART-001"], + "cjs-require": ["ART-001"], + "subpath-exports": ["ART-001"], + "bin-help-exit-zero": ["ART-001"], + "bin-version-matches-package": ["ART-001", "VER-001"], + "bin-help-banner-version-consistent": ["ART-001", "VER-001"], + "declared-dep-ranges-pinned": ["SUPPLY-001", "VER-001"], + "mcp-server-lists-tools": ["ART-001"], + "mcp-serverinfo-version-matches-package": ["ART-001", "VER-001"], + "mcp-advertised-tools-are-served": ["ART-001"], + "py-import-module": ["ART-001"], + "py-no-stdlib-shadow": ["ART-001"] + } +} diff --git a/scripts/ga/cleanroom-targets.mjs b/scripts/ga/cleanroom-targets.mjs new file mode 100644 index 0000000..42b0698 --- /dev/null +++ b/scripts/ga/cleanroom-targets.mjs @@ -0,0 +1,154 @@ +// Per-ecosystem target runners: stand up the clean room, install the published artifact, then +// hand a context to the checks. Nothing here reads the repository checkout. + +import { createHash } from 'node:crypto'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { CHECKS } from './cleanroom-checks.mjs'; +import { + PUBLIC_NPM, bad, fetchJson, installedFile, installedManifest, npmCleanRoom, npmEncode, ok, run, +} from './cleanroom-util.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +export async function runNpmTarget(target, args) { + const pkg = target.package; + const latest = await fetchJson(`${PUBLIC_NPM}/${npmEncode(pkg)}/latest`); + const version = args.versions[pkg] || latest.version; + // `version` is encoded for the same reason the package name is: it can come from the + // `workflow_dispatch` versions input, so it is not repo-controlled and must not be able to + // introduce a path separator. The PyPI path below already encodes both halves. + const packument = version === latest.version ? latest : await fetchJson(`${PUBLIC_NPM}/${npmEncode(pkg)}/${npmEncode(version)}`); + + const result = { + id: target.id, + ecosystem: 'npm', + package: pkg, + version, + registry: PUBLIC_NPM, + integrity: packument?.dist?.integrity || null, + resolved_from: args.versions[pkg] ? 'explicit --versions pin' : 'registry dist-tag `latest`', + checks: [], + }; + + const cr = npmCleanRoom(pkg, version); + result.clean_room = cr.room; + if (cr.failed) { + result.checks.push(bad('install', `${cr.failed}: ${(cr.install.stderr || cr.install.stdout || cr.install.error || '').trim().slice(0, 600)}`)); + return result; + } + const manifest = installedManifest(cr.room, pkg); + if (!manifest) { + result.checks.push(bad('install', `install reported success but ${pkg} is absent from node_modules`)); + return result; + } + result.installed_version = manifest.version; + result.checks.push(ok('install', `installed ${pkg}@${manifest.version} from ${PUBLIC_NPM} into a clean directory`)); + if (manifest.version !== version) { + result.checks.push(bad('install-version-matches-request', `requested ${version} but node_modules contains ${manifest.version}`)); + } + + let mcpCache = null; + const ctx = { + target, pkg, version, manifest, packument, room: cr.room, env: cr.env, + binPath() { + const b = manifest.bin; + const rel = typeof b === 'string' ? b : b && (b[target.bin] || Object.values(b)[0]); + return rel ? installedFile(cr.room, pkg, ...rel.split('/')) : null; + }, + async mcpProbe() { + if (mcpCache) return mcpCache; + const entry = ctx.binPath(); + if (!entry) { mcpCache = { ok: false, error: `no server entrypoint for ${pkg}` }; return mcpCache; } + const r = run(process.execPath, [join(HERE, 'mcp-stdio-probe.mjs'), entry], { cwd: cr.room, env: cr.env, timeout: 120000 }); + const line = r.stdout.trim().split('\n').filter(Boolean).pop(); + try { mcpCache = JSON.parse(line); } + catch { mcpCache = { ok: false, error: `unparseable probe output (exit ${r.status}): ${(r.stdout + r.stderr).slice(0, 300)}` }; } + return mcpCache; + }, + }; + + for (const name of target.checks) { + const fn = CHECKS[name]; + if (!fn) { result.checks.push(bad(name, 'check not implemented in cleanroom-checks.mjs')); continue; } + try { result.checks.push(await fn(ctx)); } + catch (e) { result.checks.push(bad(name, `check threw ${e?.name}: ${String(e?.message).slice(0, 300)}`)); } + } + return result; +} + +export async function runPypiTarget(target, args) { + const name = target.package; + const meta = await fetchJson(`https://pypi.org/pypi/${encodeURIComponent(name)}/json`); + const version = args.versions[name] || meta.info.version; + const files = version === meta.info.version + ? meta.urls + : (await fetchJson(`https://pypi.org/pypi/${encodeURIComponent(name)}/${encodeURIComponent(version)}/json`)).urls; + + const result = { + id: target.id, + ecosystem: 'pypi', + package: name, + version, + registry: 'https://pypi.org', + resolved_from: args.versions[name] ? 'explicit --versions pin' : 'PyPI info.version', + checks: [], + }; + + const wheel = files.find((f) => f.packagetype === 'bdist_wheel') || files.find((f) => f.packagetype === 'sdist'); + if (!wheel) { result.checks.push(bad('download', `PyPI serves no wheel or sdist for ${name}@${version}`)); return result; } + + const room = mkdtempSync(join(tmpdir(), 'wave-cleanroom-py-')); + result.clean_room = room; + const wheelPath = join(room, wheel.filename); // pip rejects a renamed wheel — keep the real filename + const bytes = Buffer.from(await (await fetch(wheel.url)).arrayBuffer()); + writeFileSync(wheelPath, bytes); + const sha = createHash('sha256').update(bytes).digest('hex'); + result.artifact = { filename: wheel.filename, url: wheel.url, sha256: sha }; + if (wheel.digests?.sha256 && wheel.digests.sha256 !== sha) { + result.checks.push(bad('download', `downloaded ${wheel.filename} sha256 ${sha} != PyPI-declared ${wheel.digests.sha256}`)); + return result; + } + result.checks.push(ok('download', `downloaded ${wheel.filename} from PyPI, sha256 ${sha} matches the declared digest`)); + + const venv = join(room, 'venv'); + const mk = run(args.python, ['-m', 'venv', venv], { cwd: room, timeout: 300000 }); + if (mk.status !== 0) { + result.checks.push(bad('venv', `could not create a clean venv with ${args.python}: ${(mk.stderr || mk.stdout || mk.error || '').trim().slice(0, 400)}`)); + return result; + } + const py = join(venv, 'bin', 'python'); + const inst = run(py, ['-m', 'pip', 'install', '-q', '--disable-pip-version-check', '--no-input', wheelPath], { cwd: room, timeout: 600000 }); + if (inst.status !== 0) { + result.checks.push(bad('install', `pip install of the downloaded wheel failed: ${(inst.stderr || inst.stdout).trim().slice(0, 600)}`)); + return result; + } + result.checks.push(ok('install', `pip installed the downloaded wheel into a fresh venv (${args.python})`)); + + // cwd is the throwaway room, never the repo: a checkout on sys.path could satisfy an import the + // published wheel is supposed to satisfy — exactly the illusion this suite exists to destroy. + // cleanroom_python_assert.py re-verifies that independently and reports it as its own check. + const argv = [join(HERE, 'cleanroom_python_assert.py'), '--dist', name, '--module', target.import_module]; + if (target.import_symbol) argv.push('--symbol', target.import_symbol); + const probe = run(py, argv, { cwd: room, timeout: 180000 }); + let parsed; + try { parsed = JSON.parse(probe.stdout.trim().split('\n').filter(Boolean).pop()); } + catch { + result.checks.push(bad('py-probe', `assertion script produced no parseable JSON (exit ${probe.status}): ${(probe.stdout + probe.stderr).slice(0, 400)}`)); + return result; + } + + result.python = parsed.python; + result.top_level = parsed.top_level; + const wanted = new Set(target.checks); + for (const c of parsed.checks) { + if (c.name === 'cleanroom-isolation' || wanted.has(c.name)) result.checks.push(c); + } + for (const want of target.checks) { + if (!parsed.checks.some((c) => c.name === want)) result.checks.push(bad(want, 'check not produced by cleanroom_python_assert.py')); + } + return result; +} diff --git a/scripts/ga/cleanroom-util.mjs b/scripts/ga/cleanroom-util.mjs new file mode 100644 index 0000000..97fedbb --- /dev/null +++ b/scripts/ga/cleanroom-util.mjs @@ -0,0 +1,100 @@ +// Shared primitives for the registry clean-room suite: process execution, registry fetches, +// check-result constructors, and the isolated-npm environment. +// +// The npm isolation is the load-bearing part. On a developer machine `@wave-av:registry` often +// points at a private GitHub registry, so an install that *looks* like "from npm" can quietly +// exercise a different artifact than customers receive. Every npm invocation here runs against a +// freshly generated user-config with no auth, no ambient scope override, and a private cache. + +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +export const PUBLIC_NPM = 'https://registry.npmjs.org'; + +export function run(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { + encoding: 'utf8', + timeout: opts.timeout ?? 600000, + maxBuffer: 32 * 1024 * 1024, + ...opts, + }); + return { + status: r.status, + signal: r.signal, + stdout: r.stdout || '', + stderr: r.stderr || '', + error: r.error ? String(r.error.message) : null, + }; +} + +export async function fetchJson(url) { + const res = await fetch(url, { headers: { 'user-agent': 'wave-ga-registry-cleanroom/1.0' } }); + if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`); + return res.json(); +} + +export function ok(name, detail) { return { name, ok: true, detail }; } +export function bad(name, detail) { return { name, ok: false, detail }; } + +/** + * Encode one value as a SINGLE npm-registry URL path segment. + * + * `encodeURIComponent`, not a hand-rolled `replace`. `pkg.replace('/', '%2f')` — the previous + * implementation — escapes only the FIRST occurrence, because a string (rather than a global regex) + * first argument replaces once. Every later separator survives into the URL as a real path + * separator: `'a/../../x'.replace('/', '%2f')` is `'a%2f../../x'`, so the fetch resolves against a + * different registry endpoint than the caller asked for. That matters here because the values are + * not all repo-controlled — `--versions` pins arrive from the `workflow_dispatch` input via + * `CLEANROOM_VERSIONS` — and because a gate that can be steered onto the wrong endpoint is a gate + * that can be made to report on an artifact nobody installs. + * + * The platform primitive escapes every occurrence and every other URL meta-character, and is the + * encoding the PyPI path in this suite already uses. It is the identity function for ordinary + * semver, including prereleases. + */ +export function npmEncode(value) { return encodeURIComponent(value); } + +/** Install one package from the PUBLIC npm registry into a throwaway directory. */ +export function npmCleanRoom(pkgName, version) { + const room = mkdtempSync(join(tmpdir(), 'wave-cleanroom-npm-')); + const userConfig = join(room, 'npm-userconfig'); + const globalConfig = join(room, 'npm-globalconfig'); + writeFileSync(userConfig, [ + `registry=${PUBLIC_NPM}/`, + `@wave-av:registry=${PUBLIC_NPM}/`, + 'audit=false', + 'fund=false', + 'update-notifier=false', + '', + ].join('\n')); + writeFileSync(globalConfig, ''); // npm refuses the same path for user and global config + + const env = { + ...process.env, + npm_config_userconfig: userConfig, + npm_config_globalconfig: globalConfig, + npm_config_cache: join(room, 'npm-cache'), + npm_config_registry: `${PUBLIC_NPM}/`, + NO_UPDATE_NOTIFIER: '1', + }; + + // Lifecycle scripts stay ENABLED on purpose: a customer's `npm install` runs them, so an + // artifact whose postinstall breaks is broken in the field and this suite must see it. + const init = run('npm', ['init', '-y'], { cwd: room, env }); + if (init.status !== 0) return { room, env, install: init, failed: 'npm init failed' }; + + const install = run('npm', ['install', '--no-audit', '--no-fund', '--loglevel=error', `${pkgName}@${version}`], { cwd: room, env }); + return { room, env, install, failed: install.status === 0 ? null : 'npm install failed' }; +} + +export function installedManifest(room, pkgName) { + const p = join(room, 'node_modules', ...pkgName.split('/'), 'package.json'); + return existsSync(p) ? JSON.parse(readFileSync(p, 'utf8')) : null; +} + +export function installedFile(room, pkgName, ...rel) { + const p = join(room, 'node_modules', ...pkgName.split('/'), ...rel); + return existsSync(p) ? p : null; +} diff --git a/scripts/ga/cleanroom_python_assert.py b/scripts/ga/cleanroom_python_assert.py new file mode 100644 index 0000000..de0345b --- /dev/null +++ b/scripts/ga/cleanroom_python_assert.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Clean-room assertions for a PyPI artifact, run INSIDE a throwaway venv. + +Invoked by scripts/ga/registry-cleanroom.mjs as: + /bin/python cleanroom_python_assert.py --dist wave-sdk --module wave_sdk --symbol Wave + +It must be run with a cwd that contains no checkout of this repository, so that a source +tree can never satisfy an import the published wheel is supposed to satisfy. The script +re-verifies that itself and refuses to report a pass if the repo is importable. + +Prints one JSON object on stdout. Exit code is always 0 — the caller reads `checks` +and decides. (Exiting nonzero here would be indistinguishable from an interpreter crash.) + +Two distinct failure modes are separated on purpose, because they are opposite bugs: + + py-import-module the documented import must WORK from the published artifact + py-no-stdlib-shadow the distribution must not claim a top-level name that collides + with a CPython stdlib module. A collision is a latent disaster in + both directions: on a path where the dist wins, every consumer of + the stdlib module silently gets the wrong library; on a path where + the stdlib wins (the usual one, since the stdlib dir precedes + site-packages), the SDK itself becomes unreachable. +""" +from __future__ import annotations + +import argparse +import importlib +import json +import os +import sys +import sysconfig + + +def realpath(p: str) -> str: + return os.path.realpath(p) if p else "" + + +def check(name: str, ok: bool, detail: str) -> dict: + return {"name": name, "ok": bool(ok), "detail": detail} + + +def dist_top_level(dist_name: str) -> list[str]: + """Top-level import names the installed distribution claims. + + top_level.txt is the cheap answer but is absent from many modern wheels, so fall back + to deriving the set from RECORD. Returns [] only when the distribution is missing. + """ + import importlib.metadata as md + + try: + dist = md.distribution(dist_name) + except md.PackageNotFoundError: + return [] + + raw = None + try: + raw = dist.read_text("top_level.txt") + except Exception: + raw = None + if raw: + return sorted({line.strip() for line in raw.splitlines() if line.strip()}) + + tops: set[str] = set() + for f in dist.files or []: + parts = str(f).split("/") + if not parts or parts[0].endswith(".dist-info") or parts[0].endswith(".data"): + continue + if len(parts) > 1: + tops.add(parts[0]) + elif parts[0].endswith(".py"): + tops.add(parts[0][:-3]) + return sorted(tops) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--dist", required=True, help="installed distribution name, e.g. wave-sdk") + ap.add_argument("--module", required=True, help="documented import name, e.g. wave_sdk") + ap.add_argument("--symbol", default=None, help="symbol the import must expose, e.g. Wave") + args = ap.parse_args() + + stdlib_dir = realpath(sysconfig.get_paths()["stdlib"]) + site_dirs = [realpath(p) for p in sys.path if "site-packages" in p] + checks: list[dict] = [] + + # Guard: a repo checkout on sys.path would make this whole run meaningless. + repo_marker_on_path = [ + p for p in sys.path + if p and os.path.isdir(os.path.join(p, "sdk-python", "wave")) + ] + checks.append(check( + "cleanroom-isolation", + not repo_marker_on_path, + "no repo checkout on sys.path" if not repo_marker_on_path + else f"REPO ON sys.path: {repo_marker_on_path} — results would be untrustworthy", + )) + + # ---- py-import-module ------------------------------------------------------------- + try: + mod = importlib.import_module(args.module) + where = realpath(getattr(mod, "__file__", "") or "") + from_site = any(where.startswith(s + os.sep) for s in site_dirs) + if args.symbol and not hasattr(mod, args.symbol): + checks.append(check( + "py-import-module", False, + f"`import {args.module}` succeeded but has no attribute " + f"`{args.symbol}` (resolved to {where or ''})", + )) + elif not from_site: + checks.append(check( + "py-import-module", False, + f"`{args.module}` resolved to {where}, which is NOT in site-packages — " + f"the published artifact did not provide it", + )) + else: + sym = f".{args.symbol}" if args.symbol else "" + checks.append(check( + "py-import-module", True, + f"`from {args.module} import {args.symbol}` OK" if args.symbol + else f"`import {args.module}`{sym} OK -> {where}", + )) + except Exception as e: # ImportError and anything the package raises at import time + checks.append(check( + "py-import-module", False, + f"`import {args.module}` raised {type(e).__name__}: {str(e)[:200]}", + )) + + # ---- py-no-stdlib-shadow ---------------------------------------------------------- + tops = dist_top_level(args.dist) + stdlib_names = set(getattr(sys, "stdlib_module_names", set())) + collisions = sorted(t for t in tops if t in stdlib_names) + if not tops: + checks.append(check( + "py-no-stdlib-shadow", False, + f"distribution `{args.dist}` is not installed — cannot determine top-level names", + )) + elif collisions: + detail_parts = [] + for c in collisions: + try: + m = importlib.import_module(c) + w = realpath(getattr(m, "__file__", "") or "") + winner = "stdlib" if w.startswith(stdlib_dir + os.sep) else "the distribution" + detail_parts.append(f"`{c}` (import resolves to {winner}: {w})") + except Exception: + detail_parts.append(f"`{c}` (unimportable)") + checks.append(check( + "py-no-stdlib-shadow", False, + f"distribution `{args.dist}` claims top-level name(s) that collide with the " + f"CPython stdlib: " + "; ".join(detail_parts), + )) + else: + checks.append(check( + "py-no-stdlib-shadow", True, + f"top-level names {tops} do not collide with the stdlib", + )) + + print(json.dumps({ + "dist": args.dist, + "module": args.module, + "python": sys.version.split()[0], + "top_level": tops, + "stdlib_dir": stdlib_dir, + "site_packages": site_dirs, + "checks": checks, + })) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ga/mcp-stdio-probe.mjs b/scripts/ga/mcp-stdio-probe.mjs new file mode 100644 index 0000000..f73ba45 --- /dev/null +++ b/scripts/ga/mcp-stdio-probe.mjs @@ -0,0 +1,124 @@ +#!/usr/bin/env node +// Minimal MCP stdio client — initialize -> notifications/initialized -> tools/list. +// +// Deliberately speaks raw JSON-RPC over stdio instead of importing @modelcontextprotocol/sdk: +// the probe must be able to contradict the artifact under test, so it shares no code with it. +// Prints a single JSON object on stdout and exits 0; on failure prints {"ok":false,...} and +// exits 1. The caller (registry-cleanroom.mjs) parses that object. +// +// Usage: node mcp-stdio-probe.mjs + +import { spawn } from 'node:child_process'; + +const RPC_TIMEOUT_MS = Number(process.env.MCP_PROBE_TIMEOUT_MS || 30000); +const entry = process.argv[2]; + +function emit(obj) { + process.stdout.write(JSON.stringify(obj) + '\n'); +} + +if (!entry) { + emit({ ok: false, error: 'usage: mcp-stdio-probe.mjs ' }); + process.exit(1); +} + +const child = spawn(process.execPath, [entry], { + stdio: ['pipe', 'pipe', 'pipe'], + env: { + ...process.env, + // A syntactically plausible but non-functional key. tools/list must not require a live + // credential — if it does, that is itself a defect worth failing on. No real secret is + // used and no authenticated request is expected to succeed. + WAVE_API_KEY: process.env.WAVE_CLEANROOM_API_KEY || 'wave_cleanroom_not_a_real_key', + }, +}); + +let stdoutBuf = ''; +let stderrBuf = ''; +const pending = new Map(); + +child.stderr.on('data', (d) => { + stderrBuf += d.toString(); + if (stderrBuf.length > 8000) stderrBuf = stderrBuf.slice(-8000); +}); + +child.stdout.on('data', (d) => { + stdoutBuf += d.toString(); + let idx; + while ((idx = stdoutBuf.indexOf('\n')) >= 0) { + const line = stdoutBuf.slice(0, idx).trim(); + stdoutBuf = stdoutBuf.slice(idx + 1); + if (!line) continue; + let msg; + try { + msg = JSON.parse(line); + } catch { + continue; // servers sometimes log non-JSON to stdout; ignore rather than crash + } + const resolve = msg && msg.id != null ? pending.get(msg.id) : undefined; + if (resolve) { + pending.delete(msg.id); + resolve(msg); + } + } +}); + +let childExit = null; +child.on('exit', (code, signal) => { + childExit = { code, signal }; + for (const [, resolve] of pending) resolve({ error: { message: `server exited (code=${code} signal=${signal})` } }); + pending.clear(); +}); + +let nextId = 1; +function rpc(method, params) { + const id = nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`timeout after ${RPC_TIMEOUT_MS}ms waiting for ${method}`)); + }, RPC_TIMEOUT_MS); + pending.set(id, (m) => { + clearTimeout(timer); + resolve(m); + }); + child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n'); + }); +} + +function notify(method, params) { + child.stdin.write(JSON.stringify({ jsonrpc: '2.0', method, params }) + '\n'); +} + +function fail(message) { + emit({ ok: false, error: message, stderr: stderrBuf.slice(0, 2000), exit: childExit }); + try { child.kill('SIGKILL'); } catch { /* already gone */ } + process.exit(1); +} + +try { + const init = await rpc('initialize', { + protocolVersion: process.env.MCP_PROTOCOL_VERSION || '2025-06-18', + capabilities: {}, + clientInfo: { name: 'wave-ga-cleanroom', version: '1.0.0' }, + }); + if (init.error) fail('initialize failed: ' + JSON.stringify(init.error).slice(0, 300)); + + notify('notifications/initialized', {}); + + const list = await rpc('tools/list', {}); + if (list.error) fail('tools/list failed: ' + JSON.stringify(list.error).slice(0, 300)); + + const tools = (list.result && list.result.tools) || []; + emit({ + ok: true, + serverInfo: (init.result && init.result.serverInfo) || null, + protocolVersion: (init.result && init.result.protocolVersion) || null, + toolCount: tools.length, + toolNames: tools.map((t) => t.name).sort(), + }); + try { child.kill('SIGKILL'); } catch { /* already gone */ } + process.exit(0); +} catch (err) { + fail(String((err && err.message) || err)); +} diff --git a/scripts/ga/registry-cleanroom.mjs b/scripts/ga/registry-cleanroom.mjs new file mode 100644 index 0000000..1e9375b --- /dev/null +++ b/scripts/ga/registry-cleanroom.mjs @@ -0,0 +1,196 @@ +#!/usr/bin/env node +/** + * registry-cleanroom — GA acceptance for what the PUBLIC REGISTRIES actually serve. + * + * WHY THIS EXISTS + * --------------- + * Every artifact regression in the pre-GA audit reached users while CI was green, because CI was + * testing the repository and the registry was serving something else. A green source branch cannot + * certify a package already published to npm or PyPI. This suite therefore never reads the + * checkout: it installs from the public registry into a throwaway directory or venv, with no repo + * on any module path, no `npm link`, no `pip install -e`, and a freshly generated npm user-config + * so a developer's scoped-registry override or auth token cannot leak in. + * + * It is built to FAIL when the registry is broken. A green run against a broken registry is the one + * outcome worse than having no suite at all. + * + * OUTPUT + * /cleanroom-report.json full detail: resolved versions, digests, per-check results + * /ga-evidence.json WAVE-GA-gate-spec-v1.0.0 evidence fragment, keyed by criterion + * + * EXIT CODES + * 0 every check of every selected target passed + * 1 the gate ran and something failed (a real artifact defect) + * 2 the gate could not run (never to be read as a pass) + * + * USAGE + * node scripts/ga/registry-cleanroom.mjs [--out-dir DIR] [--only id,id] [--python EXE] + * [--versions '@wave-av/cli=1.0.9,wave-sdk=2.1.0'] + * `--versions` (or CLEANROOM_VERSIONS) pins the exact published version to accept, which is how + * the release job tests the versions it just published instead of whatever `latest` points at. + */ + +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { runNpmTarget, runPypiTarget } from './cleanroom-targets.mjs'; +import { bad, run } from './cleanroom-util.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, '..', '..'); + +function parseVersionPins(sink, raw) { + for (const pair of String(raw).split(',')) { + const eq = pair.lastIndexOf('='); + const key = eq > 0 ? pair.slice(0, eq).trim() : ''; + if (key && !(key in sink)) sink[key] = pair.slice(eq + 1).trim(); + } +} + +function parseArgs(argv) { + const out = { + outDir: join(REPO_ROOT, 'ga-out'), + only: null, + python: process.env.CLEANROOM_PYTHON || 'python3', + versions: {}, + }; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + if (a === '--out-dir') out.outDir = resolve(argv[++i]); + else if (a === '--only') out.only = new Set(argv[++i].split(',').map((s) => s.trim()).filter(Boolean)); + else if (a === '--python') out.python = argv[++i]; + else if (a === '--versions') parseVersionPins(out.versions, argv[++i]); + else throw new Error(`unknown argument: ${a}`); + } + if (process.env.CLEANROOM_VERSIONS) parseVersionPins(out.versions, process.env.CLEANROOM_VERSIONS); + return out; +} + +/** + * Fold per-check results into per-criterion evidence. Structural checks (install/download/venv) + * carry no explicit mapping and belong to ART-001: an artifact that will not install has failed + * "published artifacts install" by definition. + */ +function buildEvidence(spec, results, revision) { + const map = spec.criteria_map || {}; + const byCriterion = new Map(); + for (const r of results) { + for (const c of r.checks) { + for (const id of map[c.name] || ['ART-001']) { + if (!byCriterion.has(id)) byCriterion.set(id, []); + byCriterion.get(id).push({ package: `${r.package}@${r.version}`, check: c.name, ok: c.ok, detail: c.detail }); + } + } + } + + // Fingerprint deliberately excludes timestamps, temp paths and durations, per the gate spec's + // idempotency rules: two runs observing the same artifacts must produce the same digest. + const fingerprint = createHash('sha256').update(JSON.stringify( + results + .map((r) => ({ + id: r.id, + package: r.package, + version: r.version, + digest: r.integrity || r.artifact?.sha256 || null, + checks: r.checks.map((c) => [c.name, c.ok]).sort(), + })) + .sort((a, b) => a.id.localeCompare(b.id)), + )).digest('hex'); + + const verifiedAt = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); + return { + fingerprint, + verifiedAt, + evidence: { + spec_version: spec.spec_version || '1.0.0', + repository: spec.repository || 'wave-av/sdks', + revision, + results: [...byCriterion.keys()].sort().map((id) => { + const rows = byCriterion.get(id); + const failures = rows.filter((r) => !r.ok); + return { + criterion_id: id, + status: failures.length === 0 ? 'pass' : 'fail', + command: 'node scripts/ga/registry-cleanroom.mjs', + evidence_sha256: fingerprint, + evidence_uri: 'ci://wave-av/sdks/.github/workflows/registry-cleanroom.yml#cleanroom-report.json', + verified_at: verifiedAt, + targets_observed: [...new Set(rows.map((r) => r.package))].sort(), + failing_checks: failures.map((f) => `${f.package}: ${f.check} — ${f.detail}`), + }; + }), + }, + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const spec = JSON.parse(readFileSync(join(HERE, 'cleanroom-targets.json'), 'utf8')); + const targets = spec.targets.filter((t) => !args.only || args.only.has(t.id)); + if (targets.length === 0) { + throw new Error(`--only matched no targets (known: ${spec.targets.map((t) => t.id).join(', ')})`); + } + + const revision = (run('git', ['rev-parse', 'HEAD'], { cwd: REPO_ROOT }).stdout || '').trim() + || process.env.GITHUB_SHA || 'unknown'; + const startedAt = new Date().toISOString(); + + process.stdout.write(`registry clean-room acceptance — ${targets.length} target(s), repo revision ${revision.slice(0, 12)}\n`); + process.stdout.write('installing ONLY from public registries; no checkout on any module path\n\n'); + + const results = []; + for (const t of targets) { + process.stdout.write(`-- ${t.id} (${t.ecosystem}: ${t.package})\n`); + let r; + try { + r = t.ecosystem === 'npm' ? await runNpmTarget(t, args) : await runPypiTarget(t, args); + } catch (e) { + r = { + id: t.id, ecosystem: t.ecosystem, package: t.package, version: 'unresolved', + checks: [bad('resolve', `${e?.name}: ${String(e?.message).slice(0, 300)}`)], + }; + } + for (const c of r.checks) process.stdout.write(` ${c.ok ? 'PASS' : 'FAIL'} ${c.name}: ${c.detail}\n`); + process.stdout.write('\n'); + results.push(r); + } + + const { evidence, fingerprint, verifiedAt } = buildEvidence(spec, results, revision); + mkdirSync(args.outDir, { recursive: true }); + writeFileSync(join(args.outDir, 'cleanroom-report.json'), `${JSON.stringify({ + schema: 'wave-registry-cleanroom/1', + spec_version: spec.spec_version || '1.0.0', + repository: spec.repository || 'wave-av/sdks', + revision, + started_at: startedAt, + finished_at: verifiedAt, + evidence_sha256: fingerprint, + runner: { node: process.version, platform: process.platform, python: args.python }, + version_pins: args.versions, + targets: results, + }, null, 2)}\n`); + writeFileSync(join(args.outDir, 'ga-evidence.json'), `${JSON.stringify(evidence, null, 2)}\n`); + + process.stdout.write(`${'-'.repeat(78)}\n`); + for (const row of evidence.results) { + process.stdout.write(`${row.criterion_id}: ${row.status.toUpperCase()} (${row.targets_observed.length} artifacts observed)\n`); + } + process.stdout.write(`\nevidence fingerprint: ${fingerprint}\n`); + process.stdout.write(`wrote ${join(args.outDir, 'cleanroom-report.json')} and ${join(args.outDir, 'ga-evidence.json')}\n`); + + const failures = results.flatMap((r) => r.checks.filter((c) => !c.ok).map((c) => `${r.package}@${r.version} ${c.name}`)); + if (failures.length > 0) { + process.stdout.write(`\nREGISTRY CLEAN-ROOM FAILED: ${failures.length} check(s)\n`); + for (const f of failures) process.stdout.write(` - ${f}\n`); + process.exitCode = 1; + return; + } + process.stdout.write('\nregistry clean-room: all checks passed\n'); +} + +main().catch((e) => { + process.stderr.write(`registry-cleanroom could not run: ${e?.stack || e}\n`); + process.exit(2); // distinct from 1 so CI can tell "gate failed" from "gate never ran" +});