diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1c585ff..fd6609f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,13 +89,11 @@ jobs: shell: bash run: | set -euo pipefail - FERN_VERSION="$(node -p "require('./fern/fern.config.json').version")" - if [[ ! "$FERN_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error file=fern/fern.config.json::Invalid Fern CLI version: $FERN_VERSION" - exit 1 - fi - npm install --global "fern-api@$FERN_VERSION" - fern check + # npm ci reproduces fern/package-lock.json exactly. tests/test_ci_workflows.py + # asserts fern/package.json and fern/fern.config.json name the same version. + npm ci --prefix fern --ignore-scripts --omit=optional + FERN_VERSION="$(node -p "require('./fern/package.json').dependencies['fern-api']")" + ./fern/node_modules/.bin/fern check { echo "### Documentation-only CI" echo diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index d0348701..bd11753e 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -23,6 +23,7 @@ jobs: with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false - name: Verify DCO sign-off on every commit env: diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index c2f723c6..43a1d7c0 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -5,18 +5,31 @@ on: branches: - main +permissions: + contents: read + jobs: run: runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/heads/main') && github.run_number > 1 }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" - - name: Install Fern - run: npm install -g fern-api + # npm ci reproduces fern/package-lock.json exactly -- every version and + # integrity hash -- so nothing resolved at run time reaches the step below, + # which holds FERN_TOKEN. + - name: Install the pinned Fern CLI + run: npm ci --prefix fern --ignore-scripts --omit=optional - name: Publish Docs env: FERN_TOKEN: ${{ secrets.FERN_TOKEN }} - run: fern generate --docs + run: ./fern/node_modules/.bin/fern generate --docs diff --git a/.gitignore b/.gitignore index a8d5f768..f1a4c74d 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ src/skillevaluator/tier3/reference_skills/*/evals/results/ skillevaluator-output* skillevaluator-quality* skillevaluator-rubric-eval* + +# Node modules for the pinned Fern documentation CLI (see fern/package.json) +node_modules/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 57deee3a..bc4ac74f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,47 @@ All notable changes to SkillEvaluator are documented in this file. directions at large case counts, and documents exact-rational omission markers. +### Security + +- The docs-publishing workflow, the only one that puts a secret in a job + environment, now pins `actions/checkout` to a commit SHA, declares + `permissions: contents: read`, and sets `persist-credentials: false`. + `dco.yml` no longer persists checkout credentials either. +- Docs are now published with the same Fern CLI version they are validated + with. `publish-docs.yml` derives it from `fern/fern.config.json`, as + `ci.yml` already did, instead of installing whatever `fern-api` resolved to + at run time. +- Workflow hardening guards now glob `.github/workflows/` instead of a + hardcoded two-file list, so every workflow — including any added later — must + pin each action to a commit, avoid persisting checkout credentials, and + declare a `permissions:` block. The guards also cover job-level + `permissions:` overrides and job-level reusable-workflow `uses:` references, + neither of which the step-level checks reached. +- The Fern CLI is now installed from a committed lockfile, `fern/package-lock.json`, + with `npm ci --prefix fern --ignore-scripts --omit=optional`, and both workflows + invoke `./fern/node_modules/.bin/fern` rather than a binary on `PATH`. Previously + `publish-docs.yml` ran `npm install`, which re-resolves every transitive + dependency from semver ranges on each run: pinning `fern-api`'s own version + pinned nothing beneath it, and `@scarf/scarf` — reached through `fern-api`'s + optional dependency on `@boundaryml/baml` — declares a `postinstall` script that + executed in the job holding `FERN_TOKEN`. The lockfile pins every package to an + exact version and integrity hash, `--ignore-scripts` stops lifecycle code + running, and `--omit=optional` (which npm honours for this local install, but + silently ignores for a `--global` one) leaves `fern-api` alone in the tree. + Guards now assert that no workflow resolves a Node dependency tree from the + registry, that every npm command carries `--ignore-scripts`, and that + `fern/package.json`, `fern/package-lock.json` and `fern/fern.config.json` all + name the same CLI version. The developer-facing install instructions + (`docs/README.md`, `docs/AGENTS.md`, `docs/developer-guide.mdx`) use the same + lockfile install, so a contributor or agent following them gets the tree CI runs. +- The action-pinning guard now accepts a same-repo composite action or + reusable workflow (`uses: ./...`) without requiring a commit SHA, since + GitHub always resolves a local reference from the caller's own commit and + nothing about it can float. A local reference that escapes the repository + (e.g. `./../outside`) or carries an `@ref` (a syntax local references don't + have) is still rejected, and a `docker://` reference is unaffected by the + exemption. + ## 0.2.1 - 2026-08-24 ### Added diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 04dff302..e26ab011 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -64,11 +64,12 @@ Navigation order, page titles, and slugs are defined in ## Verify before committing -Requires **Node.js 22+** and the Fern CLI (`npm install -g fern-api`). +Requires **Node.js 22+**. Install the pinned CLI from the committed lockfile with +`npm ci --prefix fern --ignore-scripts --omit=optional`. ```bash -fern check # validate docs.yml config and all links — must pass -fern docs dev # optional live preview at http://localhost:3000 +./fern/node_modules/.bin/fern check # validate docs.yml config and all links — must pass +./fern/node_modules/.bin/fern docs dev # optional live preview at http://localhost:3000 ``` `fern check` must pass; it is the same gate the site build relies on. Publishing diff --git a/docs/README.md b/docs/README.md index 4909e260..4397e358 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,14 +29,14 @@ published automatically from `main` via the Fern GitHub integration. Prerequisites: Node.js 22+ and npm 10+ (the versions the Fern CLI requires). ```bash -# Install the Fern CLI -npm install -g fern-api +# Install the pinned Fern CLI from the committed lockfile +npm ci --prefix fern --ignore-scripts --omit=optional # From the repo root, preview the site with live reload -fern docs dev +./fern/node_modules/.bin/fern docs dev # Validate the docs configuration and links -fern check +./fern/node_modules/.bin/fern check ``` `fern docs dev` serves the site at and reloads on changes diff --git a/docs/developer-guide.mdx b/docs/developer-guide.mdx index 027f7a2f..a42ac291 100644 --- a/docs/developer-guide.mdx +++ b/docs/developer-guide.mdx @@ -83,16 +83,18 @@ The documentation site is built with [Fern](https://buildwithfern.com/) from the ### Install the Fern CLI - Requires Node.js 22+ and npm 10+. + Requires Node.js 22+ and npm 10+. `fern/package-lock.json` pins the CLI and every package + below it; `npm ci` reproduces that tree exactly, and the flags keep npm from running any + install-time scripts. This is the same install CI performs. - ```bash title="Install fern-api" - npm install -g fern-api + ```bash title="Install the pinned fern-api" + npm ci --prefix fern --ignore-scripts --omit=optional ``` ### Preview with live reload ```bash title="Local docs preview" - fern docs dev + ./fern/node_modules/.bin/fern docs dev ``` The site serves at `http://localhost:3000` and reloads on changes to `.mdx` files or `fern/docs.yml`. @@ -100,7 +102,7 @@ The documentation site is built with [Fern](https://buildwithfern.com/) from the ### Validate before pushing ```bash title="Check config and links" - fern check + ./fern/node_modules/.bin/fern check ``` `fern check` must pass — it is the same gate the site build relies on. diff --git a/fern/package-lock.json b/fern/package-lock.json new file mode 100644 index 00000000..f4274e8b --- /dev/null +++ b/fern/package-lock.json @@ -0,0 +1,202 @@ +{ + "name": "skillevaluator-docs", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "skillevaluator-docs", + "version": "0.0.0", + "license": "Apache-2.0", + "dependencies": { + "fern-api": "5.66.1" + } + }, + "node_modules/@boundaryml/baml": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml/-/baml-0.219.0.tgz", + "integrity": "sha512-hE6t6G/1Td9yYN/T6E13igF06ZHD0J9dr5FH3tjCCZQplqTLBiVn4wyVoboK7ypjdHdGkg6O0vAcXrcbdwM6pA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@scarf/scarf": "^1.3.0" + }, + "bin": { + "baml": "cli.js", + "baml-cli": "cli.js" + }, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@boundaryml/baml-darwin-arm64": "0.219.0", + "@boundaryml/baml-darwin-x64": "0.219.0", + "@boundaryml/baml-linux-arm64-gnu": "0.219.0", + "@boundaryml/baml-linux-arm64-musl": "0.219.0", + "@boundaryml/baml-linux-x64-gnu": "0.219.0", + "@boundaryml/baml-linux-x64-musl": "0.219.0", + "@boundaryml/baml-win32-arm64-msvc": "0.219.0", + "@boundaryml/baml-win32-x64-msvc": "0.219.0" + } + }, + "node_modules/@boundaryml/baml-darwin-arm64": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-darwin-arm64/-/baml-darwin-arm64-0.219.0.tgz", + "integrity": "sha512-jhPN83UM+9Y99aCbzIS/3OpC7N8dOukPZO2piueIbzsmpaJrzTW+srKxYtVDE26Q9riFypOmizSs5SBFst04Fw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-darwin-x64": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-darwin-x64/-/baml-darwin-x64-0.219.0.tgz", + "integrity": "sha512-t7pnbVT3KEE3+stOrW7nATdipCWK8OK1gTV8qiD4pN8lZwGMtJpzrVvgWeycroWCaeJ8tNsQA/V2FyFU9pqjkg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-arm64-gnu": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-arm64-gnu/-/baml-linux-arm64-gnu-0.219.0.tgz", + "integrity": "sha512-a3ikRhlOdX+Lb08TaZU4k1Qb55mAHMuwArxpM1No/ltc0ylvo9EbJ4Nv0ZKemEHdngM6udagwdDxohpNzR1QTg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-arm64-musl": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-arm64-musl/-/baml-linux-arm64-musl-0.219.0.tgz", + "integrity": "sha512-KjwJL5aXf4XvwpX3RwMau19XcHDcffhbiTX+tk41kGC4LZl7UXB4FgnOBtSOMvWtHj2f4cIN81+yCZBRsMxdNg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-x64-gnu": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-x64-gnu/-/baml-linux-x64-gnu-0.219.0.tgz", + "integrity": "sha512-JWMhzx1LDfCPwDbOUQbFaPgNb7TUvLCb4+1TjRAJR+Z2Nl08RqNWUUfFctMEzsN6HrcHVOZ0xSl/hJRxN8OcTA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-x64-musl": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-x64-musl/-/baml-linux-x64-musl-0.219.0.tgz", + "integrity": "sha512-lPosn6eJqI+8EjMI71AUEb3B9uk4dUfQgerbDyu6FvNMWx+HZDPB4UWl66+OlEeMA2Gb6DUp3LLE/Jl9/KDKSg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-win32-arm64-msvc": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-win32-arm64-msvc/-/baml-win32-arm64-msvc-0.219.0.tgz", + "integrity": "sha512-djy9pn8s1Ogr3ovTDxDcOiM1YOhkBwNsBnlZDQzrIdR/LgSh/GGmrwW5Ks3gfa2NvvNOIFPccdUp0nDZZNsjfQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-win32-x64-msvc": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-win32-x64-msvc/-/baml-win32-x64-msvc-0.219.0.tgz", + "integrity": "sha512-BJkgq/qyn3BcBZt3iyfYWirZyZrKiqa1J0OTWd45P8OQ8dJhRnU6JGcpXPTXNmkryXQFY+O4HA+lQBKA5SZ7kg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@scarf/scarf": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz", + "integrity": "sha512-xxeapPiUXdZAE3che6f3xogoJPeZgig6omHEy1rIY5WVsB3H2BHNnZH+gHG6x91SCWyQCzWGsuL2Hh3ClO5/qQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true + }, + "node_modules/fern-api": { + "version": "5.66.1", + "resolved": "https://registry.npmjs.org/fern-api/-/fern-api-5.66.1.tgz", + "integrity": "sha512-GVakfUKF8Nwlc4MPDg24X2fjdfR3qM/cI5xtx+FNntqEZonDsFXFwvoxE3lAu+lZI90+N9U4+Ty0a4oyJeaF2A==", + "bin": { + "fern": "cli.cjs" + }, + "optionalDependencies": { + "@boundaryml/baml": "^0.219.0" + } + } + } +} diff --git a/fern/package.json b/fern/package.json new file mode 100644 index 00000000..3628f4c2 --- /dev/null +++ b/fern/package.json @@ -0,0 +1,10 @@ +{ + "name": "skillevaluator-docs", + "version": "0.0.0", + "private": true, + "description": "Pins the Fern CLI that validates and publishes the documentation site. Keep the fern-api version in step with fern.config.json; tests/test_ci_workflows.py asserts they agree.", + "license": "Apache-2.0", + "dependencies": { + "fern-api": "5.66.1" + } +} diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index a3a03f82..60e3e450 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import re from pathlib import Path from typing import Any @@ -37,6 +38,13 @@ def _load(name: str) -> dict[str, Any]: return yaml.load((WORKFLOWS / name).read_text(encoding="utf-8"), Loader=yaml.BaseLoader) +def _workflow_names() -> list[str]: + """Every workflow on disk, so a new one is covered the day it lands.""" + names = sorted(path.name for pattern in ("*.yml", "*.yaml") for path in WORKFLOWS.glob(pattern)) + assert names, "no workflows found" + return names + + def _assert_no_path_filter(workflow: dict[str, Any], event: str = "pull_request") -> None: trigger = workflow["on"][event] if isinstance(trigger, dict): @@ -49,7 +57,10 @@ def _runs(job: dict[str, Any]) -> str: def _all_uses(workflow: dict[str, Any]) -> list[str]: - return [step["uses"] for job in workflow["jobs"].values() for step in job.get("steps", []) if "uses" in step] + """Every action reference: step-level actions and job-level reusable workflows.""" + step_uses = [step["uses"] for job in workflow["jobs"].values() for step in job.get("steps", []) if "uses" in step] + job_uses = [job["uses"] for job in workflow["jobs"].values() if "uses" in job] + return step_uses + job_uses def _all_steps(workflow: dict[str, Any]) -> list[dict[str, Any]]: @@ -111,9 +122,8 @@ def test_ci_docs_lane_uses_the_required_python_312_context() -> None: assert node_step["if"] == DOCS_ONLY_IF assert len(node_step["uses"].split("@", 1)[1]) == 40 assert docs_step["if"] == DOCS_ONLY_IF - assert "fern/fern.config.json" in docs_step["run"] - assert 'npm install --global "fern-api@$FERN_VERSION"' in docs_step["run"] - assert "fern check" in docs_step["run"] + assert "npm ci --prefix fern --ignore-scripts --omit=optional" in docs_step["run"] + assert "./fern/node_modules/.bin/fern check" in docs_step["run"] assert "GITHUB_STEP_SUMMARY" in docs_step["run"] @@ -224,14 +234,119 @@ def test_non_pr_workflow_triggers_are_preserved() -> None: assert "workflow_dispatch" in security["on"] -def test_changed_workflows_pin_every_action_to_a_commit() -> None: - for workflow_name in ("ci.yml", "security.yml"): - for uses in _all_uses(_load(workflow_name)): - assert re.fullmatch(r"[^@]+@[0-9a-f]{40}", uses), uses +def _is_local_reference(uses: str) -> bool: + """A same-repo composite action or reusable workflow, e.g. ``./.github/actions/x``. + + GitHub always resolves these from the caller's own commit, so nothing about + them can float and there is no ``@ref`` syntax for one. A reference that is + both ``./``-prefixed and carries an ``@`` is therefore not this syntax -- + treat it as a normal reference so it still has to satisfy the SHA-pin check. + """ + return uses.startswith("./") and "@" not in uses -def test_changed_workflows_do_not_persist_checkout_credentials() -> None: - for workflow_name in ("ci.yml", "security.yml"): - checkout_steps = [step for step in _all_steps(_load(workflow_name)) if step.get("uses", "").startswith("actions/checkout@")] - assert checkout_steps - assert all(step.get("with", {}).get("persist-credentials") == "false" for step in checkout_steps) +def _local_reference_escapes_repo(uses: str) -> bool: + return ".." in Path(uses).parts + + +def test_every_workflow_pins_every_action_to_a_commit() -> None: + for workflow_name in _workflow_names(): + for uses in _all_uses(_load(workflow_name)): + if _is_local_reference(uses): + assert not _local_reference_escapes_repo(uses), f"{workflow_name}: {uses}" + continue + assert re.fullmatch(r"[^@]+@[0-9a-f]{40}", uses), f"{workflow_name}: {uses}" + + +def test_every_workflow_does_not_persist_checkout_credentials() -> None: + checkout_steps = [ + (workflow_name, step) + for workflow_name in _workflow_names() + for step in _all_steps(_load(workflow_name)) + if step.get("uses", "").startswith("actions/checkout@") + ] + assert checkout_steps + for workflow_name, step in checkout_steps: + assert step.get("with", {}).get("persist-credentials") == "false", workflow_name + + +def test_publish_docs_installs_the_fern_cli_from_the_committed_lockfile() -> None: + """The secret-bearing job runs a CLI whose whole tree was reviewed, not resolved. + + ``npm ci`` reproduces ``fern/package-lock.json`` exactly -- every version and + integrity hash -- so nothing between the commit and the run can change what + executes next to ``FERN_TOKEN``. + """ + job = _load("publish-docs.yml")["jobs"]["run"] + install_step = next((step for step in job["steps"] if "npm ci" in step.get("run", "")), None) + + assert install_step is not None, "publish-docs.yml no longer installs the Fern CLI from the lockfile" + assert "--ignore-scripts" in install_step["run"] + assert not re.search(r"fern-api@", install_step["run"]), "the version belongs in fern/package.json" + + publish_step = next(step for step in job["steps"] if "fern generate" in step.get("run", "")) + assert "fern/node_modules/.bin/fern" in publish_step["run"], "run the installed CLI, not one from PATH" + + +def test_every_workflow_declares_explicit_permissions() -> None: + """A job-level permissions: block overrides the workflow-level one, so check both. + + This rejects the write-all shorthand and the inherited-default token scope, not + every broad grant -- a granular write like `contents: write` is a legitimate + choice for a release workflow and is not this test's concern. + """ + for workflow_name in _workflow_names(): + workflow = _load(workflow_name) + assert workflow.get("permissions") is not None, ( + f"{workflow_name} inherits the repository default token scope" + ) + for scope, permissions in [("workflow", workflow["permissions"])] + [ + (f"job {job_id}", job["permissions"]) for job_id, job in workflow["jobs"].items() if "permissions" in job + ]: + assert permissions != "write-all", f"{workflow_name}: {scope}" + + +# Anchored at the start of a line so a comment mentioning npm is not a command. +NPM_COMMAND_LINE = re.compile(r"^[ \t]*npm (?:install|ci|i|add)\b.*$", re.MULTILINE) +NPM_REGISTRY_INSTALL = re.compile(r"^[ \t]*npm (?:install|i|add)\b(?!.*--package-lock-only).*$", re.MULTILINE) + + +def test_every_workflow_npm_command_ignores_lifecycle_scripts() -> None: + """A dependency must not get to run install-time code in a CI job. + + ``--ignore-scripts`` is the only half of this that a global install honours, + so it is asserted on every npm command regardless of how the tree is resolved. + """ + for workflow_name in _workflow_names(): + for step in _all_steps(_load(workflow_name)): + for line in NPM_COMMAND_LINE.findall(step.get("run", "")): + assert "--ignore-scripts" in line, f"{workflow_name}: {line.strip()}" + + +def test_no_workflow_resolves_a_node_dependency_tree_from_the_registry() -> None: + """Only ``npm ci`` against the committed lockfile may install into a job. + + ``npm install`` re-resolves every transitive dependency from semver ranges on + each run, so pinning the top-level version pins nothing beneath it. ``npm ci`` + installs exactly the versions and integrity hashes in ``fern/package-lock.json``. + """ + for workflow_name in _workflow_names(): + for step in _all_steps(_load(workflow_name)): + for line in NPM_REGISTRY_INSTALL.findall(step.get("run", "")): + raise AssertionError(f"{workflow_name}: use `npm ci` against the lockfile, not `{line.strip()}`") + + +def test_the_pinned_fern_cli_version_matches_the_fern_config() -> None: + """``fern/package.json`` and ``fern/fern.config.json`` both name a CLI version. + + They are two declarations of one fact. If they drift, docs are validated and + published by a different CLI than the one Fern itself is configured for. + """ + manifest = json.loads((ROOT / "fern" / "package.json").read_text(encoding="utf-8")) + fern_config = json.loads((ROOT / "fern" / "fern.config.json").read_text(encoding="utf-8")) + lockfile = json.loads((ROOT / "fern" / "package-lock.json").read_text(encoding="utf-8")) + + declared = manifest["dependencies"]["fern-api"] + assert declared == fern_config["version"], "fern/package.json and fern/fern.config.json disagree" + assert re.fullmatch(r"\d+\.\d+\.\d+", declared), f"pin an exact version, not {declared!r}" + assert lockfile["packages"]["node_modules/fern-api"]["version"] == declared, "lockfile is stale"