From 651a8bb118530a9a6b7db3947d43218298c77b23 Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:35:13 +0300 Subject: [PATCH 01/14] ci: harden publish-docs and dco workflows publish-docs.yml is the only workflow that puts a secret in a job environment (FERN_TOKEN), and was the least hardened of the four: - actions/checkout@v4 was the single unpinned uses: reference out of 36 in the repository. Pin it to 9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 (v7.0.0), the SHA already used 14 times here. - No permissions: block, so the job inherited the repository default GITHUB_TOKEN scope. Add permissions: contents: read, matching ci.yml, dco.yml and security.yml. - No persist-credentials: false, so the token was written to .git/config and stayed readable by every later step in the job -- including the npm install and fern generate steps. - npm install -g fern-api resolved its whole transitive tree at run time. Pin to 5.109.0. dco.yml only reads history with git log and git show, so it does not need persisted credentials either. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- .github/workflows/dco.yml | 1 + .github/workflows/publish-docs.yml | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index d0348701..bd11753e 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -23,6 +23,7 @@ jobs: with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false - name: Verify DCO sign-off on every commit env: diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index c2f723c6..cff1feeb 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -5,16 +5,21 @@ on: branches: - main +permissions: + contents: read + jobs: run: runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/heads/main') && github.run_number > 1 }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - name: Install Fern - run: npm install -g fern-api + run: npm install -g fern-api@5.109.0 - name: Publish Docs env: From a257d297ae5c1789e2226848f4cab2c099ec1e1c Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:35:13 +0300 Subject: [PATCH 02/14] test(ci): cover every workflow with the hardening guards test_changed_workflows_pin_every_action_to_a_commit and test_changed_workflows_do_not_persist_checkout_credentials each iterated the hardcoded tuple ("ci.yml", "security.yml"), so dco.yml and publish-docs.yml were never loaded and could drift from the invariant the tests exist to hold -- which is how the gaps fixed in the previous commit went unnoticed. Replace the tuple with a glob over .github/workflows/ so a workflow added later is covered the day it lands, and rename both tests to match what they now assert. Add a third guard: every workflow declares permissions: and does not use write-all. All three guards fail against the workflows as they stood before the previous commit, and pass after it. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- tests/test_ci_workflows.py | 40 ++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index a3a03f82..92c73077 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -37,6 +37,13 @@ def _load(name: str) -> dict[str, Any]: return yaml.load((WORKFLOWS / name).read_text(encoding="utf-8"), Loader=yaml.BaseLoader) +def _workflow_names() -> list[str]: + """Every workflow on disk, so a new one is covered the day it lands.""" + names = sorted(path.name for pattern in ("*.yml", "*.yaml") for path in WORKFLOWS.glob(pattern)) + assert names, "no workflows found" + return names + + def _assert_no_path_filter(workflow: dict[str, Any], event: str = "pull_request") -> None: trigger = workflow["on"][event] if isinstance(trigger, dict): @@ -224,14 +231,27 @@ def test_non_pr_workflow_triggers_are_preserved() -> None: assert "workflow_dispatch" in security["on"] -def test_changed_workflows_pin_every_action_to_a_commit() -> None: - for workflow_name in ("ci.yml", "security.yml"): +def test_every_workflow_pins_every_action_to_a_commit() -> None: + for workflow_name in _workflow_names(): for uses in _all_uses(_load(workflow_name)): - assert re.fullmatch(r"[^@]+@[0-9a-f]{40}", uses), uses - - -def test_changed_workflows_do_not_persist_checkout_credentials() -> None: - for workflow_name in ("ci.yml", "security.yml"): - checkout_steps = [step for step in _all_steps(_load(workflow_name)) if step.get("uses", "").startswith("actions/checkout@")] - assert checkout_steps - assert all(step.get("with", {}).get("persist-credentials") == "false" for step in checkout_steps) + assert re.fullmatch(r"[^@]+@[0-9a-f]{40}", uses), f"{workflow_name}: {uses}" + + +def test_every_workflow_does_not_persist_checkout_credentials() -> None: + checkout_steps = [ + (workflow_name, step) + for workflow_name in _workflow_names() + for step in _all_steps(_load(workflow_name)) + if step.get("uses", "").startswith("actions/checkout@") + ] + assert checkout_steps + for workflow_name, step in checkout_steps: + assert step.get("with", {}).get("persist-credentials") == "false", workflow_name + + +def test_every_workflow_declares_least_privilege_permissions() -> None: + for workflow_name in _workflow_names(): + workflow = _load(workflow_name) + permissions = workflow.get("permissions") + assert permissions is not None, f"{workflow_name} inherits the repository default token scope" + assert permissions != "write-all", workflow_name From 6ea9ae0078b43260b0a43e2ea162d5f75a6f10e2 Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:36:22 +0300 Subject: [PATCH 03/14] docs: note the workflow hardening in the changelog The project logs CI changes in CHANGELOG.md (see the 0.1.0 DCO entry and the 0.2.0 CI routing entries), so record both halves under Unreleased > Security. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57deee3a..db1f1b3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,18 @@ All notable changes to SkillEvaluator are documented in this file. directions at large case counts, and documents exact-rational omission markers. +### Security + +- The docs-publishing workflow, the only one that puts a secret in a job + environment, now pins `actions/checkout` to a commit SHA, declares + `permissions: contents: read`, sets `persist-credentials: false`, and pins + the `fern-api` install. `dco.yml` no longer persists checkout credentials + either. +- Workflow hardening guards now glob `.github/workflows/` instead of a + hardcoded two-file list, so every workflow — including any added later — must + pin each action to a commit, avoid persisting checkout credentials, and + declare a least-privilege `permissions:` block. + ## 0.2.1 - 2026-08-24 ### Added From 380420f343eb3951eeb77668ef788815ae8b89c4 Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:42:56 +0300 Subject: [PATCH 04/14] fix(ci): publish docs with the Fern version the repository pins The previous commit pinned publish-docs.yml to fern-api@5.109.0, which was wrong: fern/fern.config.json already pins the Fern CLI at 5.66.1, and ci.yml's docs lane derives the version from that file to run fern check. A hardcoded pin here would have validated docs with one CLI version and published them with another, and tests/test_ci_workflows.py already asserts the ci.yml side of that contract. Derive the version the same way ci.yml does, with the same format guard, and add a test asserting publish-docs never hardcodes it again. Add a pinned setup-node step so the version-parsing step does not depend on whichever Node the runner happens to ship -- ci.yml's docs lane already does this. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- .github/workflows/publish-docs.yml | 15 ++++++++++++++- tests/test_ci_workflows.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index cff1feeb..ba46d25b 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -18,8 +18,21 @@ jobs: with: persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Install Fern - run: npm install -g fern-api@5.109.0 + shell: bash + run: | + set -euo pipefail + FERN_VERSION="$(node -p "require('./fern/fern.config.json').version")" + if [[ ! "$FERN_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error file=fern/fern.config.json::Invalid Fern CLI version: $FERN_VERSION" + exit 1 + fi + npm install --global "fern-api@$FERN_VERSION" - name: Publish Docs env: diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index 92c73077..67ca6008 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -249,6 +249,20 @@ def test_every_workflow_does_not_persist_checkout_credentials() -> None: assert step.get("with", {}).get("persist-credentials") == "false", workflow_name +def test_publish_docs_installs_the_fern_version_the_repository_pins() -> None: + """Docs are checked and published by the same CLI version. + + ci.yml derives it from fern/fern.config.json; publishing must not hardcode a + second version that can drift from the one validation ran against. + """ + job = _load("publish-docs.yml")["jobs"]["run"] + install_step = next(step for step in job["steps"] if "npm install" in step.get("run", "")) + + assert "fern/fern.config.json" in install_step["run"] + assert "fern-api@$FERN_VERSION" in install_step["run"] + assert not re.search(r"fern-api@\d", install_step["run"]), "Fern CLI version is hardcoded" + + def test_every_workflow_declares_least_privilege_permissions() -> None: for workflow_name in _workflow_names(): workflow = _load(workflow_name) From c05200022f8a5125afd8094b7ceb28e60e34f1af Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:46:53 +0300 Subject: [PATCH 05/14] test(ci): close the job-level escalation holes in the workflow guards Code review found the new guards were step-scoped, and a probe workflow confirmed it: one declaring permissions: contents: read at the top, a job with permissions: write-all, and a second job with an unpinned reusable-workflow uses: at job level passed all thirteen tests. GitHub applies a job-level permissions: block over the workflow-level one, and a job-level uses: has no steps list for _all_uses to walk. Both are now covered. The same probe fails two guards after this commit. Also give the Fern install lookup a next(..., None) default so a restructured workflow reports a missing install step instead of raising StopIteration, and correct the changelog entry, which still claimed a hardcoded fern-api pin that the previous commit replaced. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- CHANGELOG.md | 13 +++++++++---- tests/test_ci_workflows.py | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db1f1b3d..e9d6e874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,13 +60,18 @@ All notable changes to SkillEvaluator are documented in this file. - The docs-publishing workflow, the only one that puts a secret in a job environment, now pins `actions/checkout` to a commit SHA, declares - `permissions: contents: read`, sets `persist-credentials: false`, and pins - the `fern-api` install. `dco.yml` no longer persists checkout credentials - either. + `permissions: contents: read`, and sets `persist-credentials: false`. + `dco.yml` no longer persists checkout credentials either. +- Docs are now published with the same Fern CLI version they are validated + with. `publish-docs.yml` derives it from `fern/fern.config.json`, as + `ci.yml` already did, instead of installing whatever `fern-api` resolved to + at run time. - Workflow hardening guards now glob `.github/workflows/` instead of a hardcoded two-file list, so every workflow — including any added later — must pin each action to a commit, avoid persisting checkout credentials, and - declare a least-privilege `permissions:` block. + declare a `permissions:` block. The guards also cover job-level + `permissions:` overrides and job-level reusable-workflow `uses:` references, + neither of which the step-level checks reached. ## 0.2.1 - 2026-08-24 diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index 67ca6008..ef4e135e 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -56,7 +56,10 @@ def _runs(job: dict[str, Any]) -> str: def _all_uses(workflow: dict[str, Any]) -> list[str]: - return [step["uses"] for job in workflow["jobs"].values() for step in job.get("steps", []) if "uses" in step] + """Every action reference: step-level actions and job-level reusable workflows.""" + step_uses = [step["uses"] for job in workflow["jobs"].values() for step in job.get("steps", []) if "uses" in step] + job_uses = [job["uses"] for job in workflow["jobs"].values() if "uses" in job] + return step_uses + job_uses def _all_steps(workflow: dict[str, Any]) -> list[dict[str, Any]]: @@ -256,16 +259,22 @@ def test_publish_docs_installs_the_fern_version_the_repository_pins() -> None: second version that can drift from the one validation ran against. """ job = _load("publish-docs.yml")["jobs"]["run"] - install_step = next(step for step in job["steps"] if "npm install" in step.get("run", "")) + install_step = next((step for step in job["steps"] if "npm install" in step.get("run", "")), None) + assert install_step is not None, "publish-docs.yml no longer installs the Fern CLI with npm" assert "fern/fern.config.json" in install_step["run"] assert "fern-api@$FERN_VERSION" in install_step["run"] assert not re.search(r"fern-api@\d", install_step["run"]), "Fern CLI version is hardcoded" def test_every_workflow_declares_least_privilege_permissions() -> None: + """A job-level permissions: block overrides the workflow-level one, so check both.""" for workflow_name in _workflow_names(): workflow = _load(workflow_name) - permissions = workflow.get("permissions") - assert permissions is not None, f"{workflow_name} inherits the repository default token scope" - assert permissions != "write-all", workflow_name + assert workflow.get("permissions") is not None, ( + f"{workflow_name} inherits the repository default token scope" + ) + for scope, permissions in [("workflow", workflow["permissions"])] + [ + (f"job {job_id}", job["permissions"]) for job_id, job in workflow["jobs"].items() if "permissions" in job + ]: + assert permissions != "write-all", f"{workflow_name}: {scope}" From 28082c75ed1f947898c9f2662c9ef0ea41460098 Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:10:40 +0300 Subject: [PATCH 06/14] fix(ci): stop the Fern install from running lifecycle scripts Maintainer review (rng1995, PR #126, P1) found that pinning fern-api's top-level version does not pin its dependency tree: npm re-resolves every transitive dependency from semver ranges on each install. fern-api@5.66.1 has an optional dependency on @boundaryml/baml@^0.219.0, which depends on @scarf/scarf@^1.3.0, which resolves to 1.4.0 and declares postinstall: node ./report.js. That script still ran in the job immediately before FERN_TOKEN was used. Pass --ignore-scripts on both npm installs (publish-docs.yml and ci.yml's docs-validation lane, which runs the identical command). fern-api declares no scripts of its own and npm links its bin natively, so nothing the CLI needs is skipped -- this alone removes the lifecycle-script execution vector. Also pass --omit=optional, which drops @boundaryml/baml and the eight native binaries it pulls in. This rests on one assumption that cannot be verified in CI: publish-docs.yml only runs on push to main, gated on github.run_number > 1, so it never executes on a pull request. Because BAML is declared optional, fern-api must already tolerate its absence at install time; the residual risk is a lazily-required docs code path. If docs publishing breaks on the merge commit, the rollback is to drop --omit=optional and keep --ignore-scripts, which retains the security property that motivated this change. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/publish-docs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1c585ff..91063e8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,7 @@ jobs: echo "::error file=fern/fern.config.json::Invalid Fern CLI version: $FERN_VERSION" exit 1 fi - npm install --global "fern-api@$FERN_VERSION" + npm install --global --ignore-scripts --omit=optional "fern-api@$FERN_VERSION" fern check { echo "### Documentation-only CI" diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index ba46d25b..4d5f58d8 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -32,7 +32,7 @@ jobs: echo "::error file=fern/fern.config.json::Invalid Fern CLI version: $FERN_VERSION" exit 1 fi - npm install --global "fern-api@$FERN_VERSION" + npm install --global --ignore-scripts --omit=optional "fern-api@$FERN_VERSION" - name: Publish Docs env: From 442ae9022bda5b1b7751d205c26ab42fb41f9b4f Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:10:53 +0300 Subject: [PATCH 07/14] test(ci): enforce --ignore-scripts, and accept validated local action refs Two changes to the workflow guards, both from the same review round: Add test_every_workflow_npm_install_ignores_lifecycle_scripts, asserting every npm install line in any workflow's run steps carries --ignore-scripts. It follows the same glob-every-workflow shape as the other guards, so a fifth workflow is covered the day it lands. --omit=optional is deliberately not asserted: it is a judgement about one CLI's optional dependency, not a repository-wide invariant, and asserting it would block a future workflow that legitimately needs optional dependencies. Confirmed non-vacuous by the previous commit: before it, this guard failed against the real, unpatched ci.yml and publish-docs.yml. Relax test_every_workflow_pins_every_action_to_a_commit so a same-repo reference (uses: ./...) no longer needs a commit SHA. GitHub always resolves a local composite action or reusable workflow from the caller's own commit, so nothing about it can float, and the guard previously rejected the only syntax GitHub supports for it (reviewer rng1995, PR #126, P2). A local reference that escapes the repository (./../outside) is still rejected, and docker://image:tag is unaffected -- a mutable container tag is not a local reference and stays rejected. The exemption lives in the guard, not in _all_uses, so a future guard can still assert something about local references specifically. Probed with throwaway workflow files (removed before this commit): ./.github/actions/x failed the guard before this change and passes after; ./../outside and docker://alpine:3.18 failed before and still fail after. Also update the one pre-existing assertion that hardcoded the old install line verbatim. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- tests/test_ci_workflows.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index ef4e135e..d285609e 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -122,7 +122,7 @@ def test_ci_docs_lane_uses_the_required_python_312_context() -> None: assert len(node_step["uses"].split("@", 1)[1]) == 40 assert docs_step["if"] == DOCS_ONLY_IF assert "fern/fern.config.json" in docs_step["run"] - assert 'npm install --global "fern-api@$FERN_VERSION"' in docs_step["run"] + assert 'npm install --global --ignore-scripts --omit=optional "fern-api@$FERN_VERSION"' in docs_step["run"] assert "fern check" in docs_step["run"] assert "GITHUB_STEP_SUMMARY" in docs_step["run"] @@ -234,9 +234,25 @@ def test_non_pr_workflow_triggers_are_preserved() -> None: assert "workflow_dispatch" in security["on"] +def _is_local_reference(uses: str) -> bool: + """A same-repo composite action or reusable workflow, e.g. ``./.github/actions/x``. + + GitHub always resolves these from the caller's own commit, so nothing about + them can float and there is no ``@ref`` to pin. + """ + return uses.startswith("./") + + +def _local_reference_escapes_repo(uses: str) -> bool: + return ".." in Path(uses).parts + + def test_every_workflow_pins_every_action_to_a_commit() -> None: for workflow_name in _workflow_names(): for uses in _all_uses(_load(workflow_name)): + if _is_local_reference(uses): + assert not _local_reference_escapes_repo(uses), f"{workflow_name}: {uses}" + continue assert re.fullmatch(r"[^@]+@[0-9a-f]{40}", uses), f"{workflow_name}: {uses}" @@ -278,3 +294,18 @@ def test_every_workflow_declares_least_privilege_permissions() -> None: (f"job {job_id}", job["permissions"]) for job_id, job in workflow["jobs"].items() if "permissions" in job ]: assert permissions != "write-all", f"{workflow_name}: {scope}" + + +NPM_INSTALL_LINE = re.compile(r"^.*\bnpm install\b.*$", re.MULTILINE) + + +def test_every_workflow_npm_install_ignores_lifecycle_scripts() -> None: + """A floating transitive dependency must not get to run install-time code. + + npm re-resolves the whole tree on every install, so pinning the top-level + package does not pin what its dependencies can execute at install time. + """ + for workflow_name in _workflow_names(): + for step in _all_steps(_load(workflow_name)): + for line in NPM_INSTALL_LINE.findall(step.get("run", "")): + assert "--ignore-scripts" in line, f"{workflow_name}: {line.strip()}" From f34172462e3763fa68df71af79c9f6ee9997e023 Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:11:01 +0300 Subject: [PATCH 08/14] docs: note the lifecycle-script and local-ref fixes in the changelog Same convention as the earlier commits in this branch: CI security changes are logged under Unreleased > Security. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9d6e874..6a479f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,21 @@ All notable changes to SkillEvaluator are documented in this file. declare a `permissions:` block. The guards also cover job-level `permissions:` overrides and job-level reusable-workflow `uses:` references, neither of which the step-level checks reached. +- The Fern CLI install in `publish-docs.yml` and `ci.yml`'s docs-validation + lane now passes `--ignore-scripts`, so a compromised package anywhere in + `fern-api`'s transitive dependency tree can no longer run lifecycle code in + the job that runs immediately before `FERN_TOKEN` is used. `--omit=optional` + also drops the tree's only optional dependency, `@boundaryml/baml`, along + with the native binaries it pulls in; pinning `fern-api`'s own version does + not pin what its dependencies resolve to on each run, since npm re-resolves + the whole tree from semver ranges every time. A guard now asserts every + workflow's `npm install` carries `--ignore-scripts`. +- The action-pinning guard now accepts a same-repo composite action or + reusable workflow (`uses: ./...`) without requiring a commit SHA, since + GitHub always resolves a local reference from the caller's own commit and + nothing about it can float. A local reference that escapes the repository + (e.g. `./../outside`) is still rejected, and a `docker://` reference is + unaffected by the exemption. ## 0.2.1 - 2026-08-24 From 6b80a9357e758866d5013ae9bc786e9aaed74e1e Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:23:29 +0300 Subject: [PATCH 09/14] test(ci): reject a local reference that also carries an @ref Required code review (mattpocock-skills:code-review, high effort, range ea9b239..HEAD) found that _is_local_reference only checked for a ./ prefix, so a malformed reference like ./.github/actions/build@main was classified as local and fully exempted from the commit-pin requirement. GitHub has no @ref syntax for a local path, so a string like this is either a typo or a misunderstanding of the syntax, not a reference nothing can float -- it should still be judged by the same pinning rule as everything else. Probed with a throwaway workflow (removed before this commit): the bogus reference passed the guard before this change and fails it after, with the pre-existing SHA-format assertion doing the rejecting since a local prefix no longer short-circuits it. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- tests/test_ci_workflows.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index d285609e..c7445db7 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -238,9 +238,11 @@ def _is_local_reference(uses: str) -> bool: """A same-repo composite action or reusable workflow, e.g. ``./.github/actions/x``. GitHub always resolves these from the caller's own commit, so nothing about - them can float and there is no ``@ref`` to pin. + them can float and there is no ``@ref`` syntax for one. A reference that is + both ``./``-prefixed and carries an ``@`` is therefore not this syntax -- + treat it as a normal reference so it still has to satisfy the SHA-pin check. """ - return uses.startswith("./") + return uses.startswith("./") and "@" not in uses def _local_reference_escapes_repo(uses: str) -> bool: From f23c42799ba0dc6c216d9126787552ac8bbc332d Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Tue, 1 Sep 2026 05:13:17 +0300 Subject: [PATCH 10/14] docs: harden the local Fern install instructions to match CI docs/README.md, docs/AGENTS.md, and docs/developer-guide.mdx all told a contributor (or an agent following docs/AGENTS.md) to run npm install -g fern-api with no flags -- worse than the workflows were before this branch's earlier commits, since the docs never even pinned a version. Add --ignore-scripts --omit=optional to all three, matching what the workflows now do, so lifecycle-script execution isn't reduced in CI while staying open on a contributor's own machine. Deliberately not adding a version pin here: fern/fern.config.json is the single source of truth the workflows derive their version from, and hardcoding a version into three docs files would reintroduce the exact drift risk an earlier commit on this branch removed. Also correct the CHANGELOG entry from two commits ago, which described the local-reference exemption's rejection list without the @ref case this branch's most recent commit closed. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- CHANGELOG.md | 10 +++++++--- docs/AGENTS.md | 2 +- docs/README.md | 2 +- docs/developer-guide.mdx | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a479f8b..c94a5414 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,13 +80,17 @@ All notable changes to SkillEvaluator are documented in this file. with the native binaries it pulls in; pinning `fern-api`'s own version does not pin what its dependencies resolve to on each run, since npm re-resolves the whole tree from semver ranges every time. A guard now asserts every - workflow's `npm install` carries `--ignore-scripts`. + workflow's `npm install` carries `--ignore-scripts`. The developer-facing + install instructions (`docs/README.md`, `docs/AGENTS.md`, + `docs/developer-guide.mdx`) carry the same two flags now, so a contributor + or agent following them locally gets the same hardening CI does. - The action-pinning guard now accepts a same-repo composite action or reusable workflow (`uses: ./...`) without requiring a commit SHA, since GitHub always resolves a local reference from the caller's own commit and nothing about it can float. A local reference that escapes the repository - (e.g. `./../outside`) is still rejected, and a `docker://` reference is - unaffected by the exemption. + (e.g. `./../outside`) or carries an `@ref` (a syntax local references don't + have) is still rejected, and a `docker://` reference is unaffected by the + exemption. ## 0.2.1 - 2026-08-24 diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 04dff302..fee644b7 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -64,7 +64,7 @@ Navigation order, page titles, and slugs are defined in ## Verify before committing -Requires **Node.js 22+** and the Fern CLI (`npm install -g fern-api`). +Requires **Node.js 22+** and the Fern CLI (`npm install -g --ignore-scripts --omit=optional fern-api`). ```bash fern check # validate docs.yml config and all links — must pass diff --git a/docs/README.md b/docs/README.md index 4909e260..e8f210d0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,7 +30,7 @@ Prerequisites: Node.js 22+ and npm 10+ (the versions the Fern CLI requires). ```bash # Install the Fern CLI -npm install -g fern-api +npm install -g --ignore-scripts --omit=optional fern-api # From the repo root, preview the site with live reload fern docs dev diff --git a/docs/developer-guide.mdx b/docs/developer-guide.mdx index 027f7a2f..0dd56d71 100644 --- a/docs/developer-guide.mdx +++ b/docs/developer-guide.mdx @@ -83,10 +83,11 @@ The documentation site is built with [Fern](https://buildwithfern.com/) from the ### Install the Fern CLI - Requires Node.js 22+ and npm 10+. + Requires Node.js 22+ and npm 10+. `--ignore-scripts --omit=optional` keeps npm from running any + package's install-time scripts, the same hardening CI applies to this install. ```bash title="Install fern-api" - npm install -g fern-api + npm install -g --ignore-scripts --omit=optional fern-api ``` ### Preview with live reload From d4b131b772b9dae6c33b744edfd738569ac7a848 Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Tue, 1 Sep 2026 05:13:25 +0300 Subject: [PATCH 11/14] test(ci): rename the permissions guard to match what it checks test_every_workflow_declares_least_privilege_permissions only rejects the write-all shorthand and a missing permissions: block; an explicit {contents: write, packages: write, id-token: write} block passes cleanly despite being just as broad. The name and docstring promised more than the assertion delivers. Rename to test_every_workflow_declares_explicit_permissions and add a docstring line stating the scope directly, so a reader doesn't infer a least-privilege guarantee that isn't there. Not strengthening the assertion: a release workflow legitimately needs a granular write, and this guard isn't meant to block that. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- tests/test_ci_workflows.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index c7445db7..d253c2bf 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -285,8 +285,13 @@ def test_publish_docs_installs_the_fern_version_the_repository_pins() -> None: assert not re.search(r"fern-api@\d", install_step["run"]), "Fern CLI version is hardcoded" -def test_every_workflow_declares_least_privilege_permissions() -> None: - """A job-level permissions: block overrides the workflow-level one, so check both.""" +def test_every_workflow_declares_explicit_permissions() -> None: + """A job-level permissions: block overrides the workflow-level one, so check both. + + This rejects the write-all shorthand and the inherited-default token scope, not + every broad grant -- a granular write like `contents: write` is a legitimate + choice for a release workflow and is not this test's concern. + """ for workflow_name in _workflow_names(): workflow = _load(workflow_name) assert workflow.get("permissions") is not None, ( From c98abd18e25bbdad4fafa729c8993f79d3f36de5 Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Tue, 1 Sep 2026 06:01:01 +0300 Subject: [PATCH 12/14] ci: install the Fern CLI from a committed lockfile, not the registry The previous commit pinned fern-api's own version and added --ignore-scripts --omit=optional. Testing that fix against npm rather than assuming it, two things turned out to be wrong: - --omit=optional is silently ignored for a --global install. Installing fern-api@5.66.1 globally with and without the flag produces byte-identical trees on npm 10.9.9 and 11.12.1; @scarf/scarf and its postinstall land either way. The flag only takes effect for a local install. - Pinning the top-level version leaves every transitive edge floating. fern-api ^0.219.0 -> @boundaryml/baml -> @scarf/scarf ^1.3.0 is re-resolved from semver on every run, so a compromised republish still reaches the job. Commit fern/package.json and fern/package-lock.json, and install with `npm ci --prefix fern --ignore-scripts --omit=optional`. npm ci reproduces the lockfile exactly -- every version and integrity hash -- so nothing resolved at run time reaches the step holding FERN_TOKEN. --omit=optional is honoured here because this is a local install, and it leaves fern-api alone in the tree. Both workflows now invoke ./fern/node_modules/.bin/fern rather than a binary on PATH, so a PATH entry cannot substitute the CLI. Verified end to end: the exact command sequence installs one package and `fern check` reports 0 errors against this repository. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- .github/workflows/ci.yml | 12 +- .github/workflows/publish-docs.yml | 17 +-- .gitignore | 3 + fern/package-lock.json | 202 +++++++++++++++++++++++++++++ fern/package.json | 10 ++ 5 files changed, 226 insertions(+), 18 deletions(-) create mode 100644 fern/package-lock.json create mode 100644 fern/package.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91063e8e..fd6609f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,13 +89,11 @@ jobs: shell: bash run: | set -euo pipefail - FERN_VERSION="$(node -p "require('./fern/fern.config.json').version")" - if [[ ! "$FERN_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error file=fern/fern.config.json::Invalid Fern CLI version: $FERN_VERSION" - exit 1 - fi - npm install --global --ignore-scripts --omit=optional "fern-api@$FERN_VERSION" - fern check + # npm ci reproduces fern/package-lock.json exactly. tests/test_ci_workflows.py + # asserts fern/package.json and fern/fern.config.json name the same version. + npm ci --prefix fern --ignore-scripts --omit=optional + FERN_VERSION="$(node -p "require('./fern/package.json').dependencies['fern-api']")" + ./fern/node_modules/.bin/fern check { echo "### Documentation-only CI" echo diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 4d5f58d8..43a1d7c0 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -23,18 +23,13 @@ jobs: with: node-version: "22" - - name: Install Fern - shell: bash - run: | - set -euo pipefail - FERN_VERSION="$(node -p "require('./fern/fern.config.json').version")" - if [[ ! "$FERN_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error file=fern/fern.config.json::Invalid Fern CLI version: $FERN_VERSION" - exit 1 - fi - npm install --global --ignore-scripts --omit=optional "fern-api@$FERN_VERSION" + # npm ci reproduces fern/package-lock.json exactly -- every version and + # integrity hash -- so nothing resolved at run time reaches the step below, + # which holds FERN_TOKEN. + - name: Install the pinned Fern CLI + run: npm ci --prefix fern --ignore-scripts --omit=optional - name: Publish Docs env: FERN_TOKEN: ${{ secrets.FERN_TOKEN }} - run: fern generate --docs + run: ./fern/node_modules/.bin/fern generate --docs diff --git a/.gitignore b/.gitignore index a8d5f768..f1a4c74d 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ src/skillevaluator/tier3/reference_skills/*/evals/results/ skillevaluator-output* skillevaluator-quality* skillevaluator-rubric-eval* + +# Node modules for the pinned Fern documentation CLI (see fern/package.json) +node_modules/ diff --git a/fern/package-lock.json b/fern/package-lock.json new file mode 100644 index 00000000..f4274e8b --- /dev/null +++ b/fern/package-lock.json @@ -0,0 +1,202 @@ +{ + "name": "skillevaluator-docs", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "skillevaluator-docs", + "version": "0.0.0", + "license": "Apache-2.0", + "dependencies": { + "fern-api": "5.66.1" + } + }, + "node_modules/@boundaryml/baml": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml/-/baml-0.219.0.tgz", + "integrity": "sha512-hE6t6G/1Td9yYN/T6E13igF06ZHD0J9dr5FH3tjCCZQplqTLBiVn4wyVoboK7ypjdHdGkg6O0vAcXrcbdwM6pA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@scarf/scarf": "^1.3.0" + }, + "bin": { + "baml": "cli.js", + "baml-cli": "cli.js" + }, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@boundaryml/baml-darwin-arm64": "0.219.0", + "@boundaryml/baml-darwin-x64": "0.219.0", + "@boundaryml/baml-linux-arm64-gnu": "0.219.0", + "@boundaryml/baml-linux-arm64-musl": "0.219.0", + "@boundaryml/baml-linux-x64-gnu": "0.219.0", + "@boundaryml/baml-linux-x64-musl": "0.219.0", + "@boundaryml/baml-win32-arm64-msvc": "0.219.0", + "@boundaryml/baml-win32-x64-msvc": "0.219.0" + } + }, + "node_modules/@boundaryml/baml-darwin-arm64": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-darwin-arm64/-/baml-darwin-arm64-0.219.0.tgz", + "integrity": "sha512-jhPN83UM+9Y99aCbzIS/3OpC7N8dOukPZO2piueIbzsmpaJrzTW+srKxYtVDE26Q9riFypOmizSs5SBFst04Fw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-darwin-x64": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-darwin-x64/-/baml-darwin-x64-0.219.0.tgz", + "integrity": "sha512-t7pnbVT3KEE3+stOrW7nATdipCWK8OK1gTV8qiD4pN8lZwGMtJpzrVvgWeycroWCaeJ8tNsQA/V2FyFU9pqjkg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-arm64-gnu": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-arm64-gnu/-/baml-linux-arm64-gnu-0.219.0.tgz", + "integrity": "sha512-a3ikRhlOdX+Lb08TaZU4k1Qb55mAHMuwArxpM1No/ltc0ylvo9EbJ4Nv0ZKemEHdngM6udagwdDxohpNzR1QTg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-arm64-musl": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-arm64-musl/-/baml-linux-arm64-musl-0.219.0.tgz", + "integrity": "sha512-KjwJL5aXf4XvwpX3RwMau19XcHDcffhbiTX+tk41kGC4LZl7UXB4FgnOBtSOMvWtHj2f4cIN81+yCZBRsMxdNg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-x64-gnu": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-x64-gnu/-/baml-linux-x64-gnu-0.219.0.tgz", + "integrity": "sha512-JWMhzx1LDfCPwDbOUQbFaPgNb7TUvLCb4+1TjRAJR+Z2Nl08RqNWUUfFctMEzsN6HrcHVOZ0xSl/hJRxN8OcTA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-x64-musl": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-x64-musl/-/baml-linux-x64-musl-0.219.0.tgz", + "integrity": "sha512-lPosn6eJqI+8EjMI71AUEb3B9uk4dUfQgerbDyu6FvNMWx+HZDPB4UWl66+OlEeMA2Gb6DUp3LLE/Jl9/KDKSg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-win32-arm64-msvc": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-win32-arm64-msvc/-/baml-win32-arm64-msvc-0.219.0.tgz", + "integrity": "sha512-djy9pn8s1Ogr3ovTDxDcOiM1YOhkBwNsBnlZDQzrIdR/LgSh/GGmrwW5Ks3gfa2NvvNOIFPccdUp0nDZZNsjfQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-win32-x64-msvc": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-win32-x64-msvc/-/baml-win32-x64-msvc-0.219.0.tgz", + "integrity": "sha512-BJkgq/qyn3BcBZt3iyfYWirZyZrKiqa1J0OTWd45P8OQ8dJhRnU6JGcpXPTXNmkryXQFY+O4HA+lQBKA5SZ7kg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@scarf/scarf": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz", + "integrity": "sha512-xxeapPiUXdZAE3che6f3xogoJPeZgig6omHEy1rIY5WVsB3H2BHNnZH+gHG6x91SCWyQCzWGsuL2Hh3ClO5/qQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true + }, + "node_modules/fern-api": { + "version": "5.66.1", + "resolved": "https://registry.npmjs.org/fern-api/-/fern-api-5.66.1.tgz", + "integrity": "sha512-GVakfUKF8Nwlc4MPDg24X2fjdfR3qM/cI5xtx+FNntqEZonDsFXFwvoxE3lAu+lZI90+N9U4+Ty0a4oyJeaF2A==", + "bin": { + "fern": "cli.cjs" + }, + "optionalDependencies": { + "@boundaryml/baml": "^0.219.0" + } + } + } +} diff --git a/fern/package.json b/fern/package.json new file mode 100644 index 00000000..3628f4c2 --- /dev/null +++ b/fern/package.json @@ -0,0 +1,10 @@ +{ + "name": "skillevaluator-docs", + "version": "0.0.0", + "private": true, + "description": "Pins the Fern CLI that validates and publishes the documentation site. Keep the fern-api version in step with fern.config.json; tests/test_ci_workflows.py asserts they agree.", + "license": "Apache-2.0", + "dependencies": { + "fern-api": "5.66.1" + } +} From 27ad87cd400c47192c96fa2435fc462dd6f548fe Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Tue, 1 Sep 2026 06:01:02 +0300 Subject: [PATCH 13/14] test(ci): require the lockfile install and keep the pinned version honest Three guards, each proven against a probe: - no workflow may resolve a Node dependency tree from the registry; only `npm ci` against the committed lockfile installs into a job. - every npm command carries --ignore-scripts. Widened from `npm install` to cover `npm ci`, and anchored to the start of a line so a comment mentioning npm is no longer matched as a command. - fern/package.json, fern/package-lock.json and fern/fern.config.json name the same CLI version. A lockfile adds a second home for that version, and this stops the two drifting -- docs validated by one CLI and published by another is the failure this PR already fixed once. Reverting ci.yml to `npm install --global` fails the first two; changing fern.config.json's version alone fails the third. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- tests/test_ci_workflows.py | 70 ++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index d253c2bf..60e3e450 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import re from pathlib import Path from typing import Any @@ -121,9 +122,8 @@ def test_ci_docs_lane_uses_the_required_python_312_context() -> None: assert node_step["if"] == DOCS_ONLY_IF assert len(node_step["uses"].split("@", 1)[1]) == 40 assert docs_step["if"] == DOCS_ONLY_IF - assert "fern/fern.config.json" in docs_step["run"] - assert 'npm install --global --ignore-scripts --omit=optional "fern-api@$FERN_VERSION"' in docs_step["run"] - assert "fern check" in docs_step["run"] + assert "npm ci --prefix fern --ignore-scripts --omit=optional" in docs_step["run"] + assert "./fern/node_modules/.bin/fern check" in docs_step["run"] assert "GITHUB_STEP_SUMMARY" in docs_step["run"] @@ -270,19 +270,22 @@ def test_every_workflow_does_not_persist_checkout_credentials() -> None: assert step.get("with", {}).get("persist-credentials") == "false", workflow_name -def test_publish_docs_installs_the_fern_version_the_repository_pins() -> None: - """Docs are checked and published by the same CLI version. +def test_publish_docs_installs_the_fern_cli_from_the_committed_lockfile() -> None: + """The secret-bearing job runs a CLI whose whole tree was reviewed, not resolved. - ci.yml derives it from fern/fern.config.json; publishing must not hardcode a - second version that can drift from the one validation ran against. + ``npm ci`` reproduces ``fern/package-lock.json`` exactly -- every version and + integrity hash -- so nothing between the commit and the run can change what + executes next to ``FERN_TOKEN``. """ job = _load("publish-docs.yml")["jobs"]["run"] - install_step = next((step for step in job["steps"] if "npm install" in step.get("run", "")), None) + install_step = next((step for step in job["steps"] if "npm ci" in step.get("run", "")), None) - assert install_step is not None, "publish-docs.yml no longer installs the Fern CLI with npm" - assert "fern/fern.config.json" in install_step["run"] - assert "fern-api@$FERN_VERSION" in install_step["run"] - assert not re.search(r"fern-api@\d", install_step["run"]), "Fern CLI version is hardcoded" + assert install_step is not None, "publish-docs.yml no longer installs the Fern CLI from the lockfile" + assert "--ignore-scripts" in install_step["run"] + assert not re.search(r"fern-api@", install_step["run"]), "the version belongs in fern/package.json" + + publish_step = next(step for step in job["steps"] if "fern generate" in step.get("run", "")) + assert "fern/node_modules/.bin/fern" in publish_step["run"], "run the installed CLI, not one from PATH" def test_every_workflow_declares_explicit_permissions() -> None: @@ -303,16 +306,47 @@ def test_every_workflow_declares_explicit_permissions() -> None: assert permissions != "write-all", f"{workflow_name}: {scope}" -NPM_INSTALL_LINE = re.compile(r"^.*\bnpm install\b.*$", re.MULTILINE) +# Anchored at the start of a line so a comment mentioning npm is not a command. +NPM_COMMAND_LINE = re.compile(r"^[ \t]*npm (?:install|ci|i|add)\b.*$", re.MULTILINE) +NPM_REGISTRY_INSTALL = re.compile(r"^[ \t]*npm (?:install|i|add)\b(?!.*--package-lock-only).*$", re.MULTILINE) -def test_every_workflow_npm_install_ignores_lifecycle_scripts() -> None: - """A floating transitive dependency must not get to run install-time code. +def test_every_workflow_npm_command_ignores_lifecycle_scripts() -> None: + """A dependency must not get to run install-time code in a CI job. - npm re-resolves the whole tree on every install, so pinning the top-level - package does not pin what its dependencies can execute at install time. + ``--ignore-scripts`` is the only half of this that a global install honours, + so it is asserted on every npm command regardless of how the tree is resolved. """ for workflow_name in _workflow_names(): for step in _all_steps(_load(workflow_name)): - for line in NPM_INSTALL_LINE.findall(step.get("run", "")): + for line in NPM_COMMAND_LINE.findall(step.get("run", "")): assert "--ignore-scripts" in line, f"{workflow_name}: {line.strip()}" + + +def test_no_workflow_resolves_a_node_dependency_tree_from_the_registry() -> None: + """Only ``npm ci`` against the committed lockfile may install into a job. + + ``npm install`` re-resolves every transitive dependency from semver ranges on + each run, so pinning the top-level version pins nothing beneath it. ``npm ci`` + installs exactly the versions and integrity hashes in ``fern/package-lock.json``. + """ + for workflow_name in _workflow_names(): + for step in _all_steps(_load(workflow_name)): + for line in NPM_REGISTRY_INSTALL.findall(step.get("run", "")): + raise AssertionError(f"{workflow_name}: use `npm ci` against the lockfile, not `{line.strip()}`") + + +def test_the_pinned_fern_cli_version_matches_the_fern_config() -> None: + """``fern/package.json`` and ``fern/fern.config.json`` both name a CLI version. + + They are two declarations of one fact. If they drift, docs are validated and + published by a different CLI than the one Fern itself is configured for. + """ + manifest = json.loads((ROOT / "fern" / "package.json").read_text(encoding="utf-8")) + fern_config = json.loads((ROOT / "fern" / "fern.config.json").read_text(encoding="utf-8")) + lockfile = json.loads((ROOT / "fern" / "package-lock.json").read_text(encoding="utf-8")) + + declared = manifest["dependencies"]["fern-api"] + assert declared == fern_config["version"], "fern/package.json and fern/fern.config.json disagree" + assert re.fullmatch(r"\d+\.\d+\.\d+", declared), f"pin an exact version, not {declared!r}" + assert lockfile["packages"]["node_modules/fern-api"]["version"] == declared, "lockfile is stale" From 8ad4932e95920ee5028c9f386309a8cae7937939 Mon Sep 17 00:00:00 2001 From: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> Date: Tue, 1 Sep 2026 06:01:02 +0300 Subject: [PATCH 14/14] docs: install the pinned Fern CLI from the lockfile, and correct the changelog docs/README.md, docs/AGENTS.md and docs/developer-guide.mdx told contributors to run `npm install -g ... fern-api` -- unpinned, and with the --omit=optional flag that a global install ignores. They now use the same lockfile install CI runs. The changelog entry claiming --omit=optional "drops the tree's only optional dependency" was wrong for the same reason and is rewritten to describe what was measured rather than what the flag name suggests. Signed-off-by: berzan93-prog <224507836+berzan93-prog@users.noreply.github.com> --- CHANGELOG.md | 29 +++++++++++++++++------------ docs/AGENTS.md | 7 ++++--- docs/README.md | 8 ++++---- docs/developer-guide.mdx | 13 +++++++------ 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c94a5414..bc4ac74f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,18 +72,23 @@ All notable changes to SkillEvaluator are documented in this file. declare a `permissions:` block. The guards also cover job-level `permissions:` overrides and job-level reusable-workflow `uses:` references, neither of which the step-level checks reached. -- The Fern CLI install in `publish-docs.yml` and `ci.yml`'s docs-validation - lane now passes `--ignore-scripts`, so a compromised package anywhere in - `fern-api`'s transitive dependency tree can no longer run lifecycle code in - the job that runs immediately before `FERN_TOKEN` is used. `--omit=optional` - also drops the tree's only optional dependency, `@boundaryml/baml`, along - with the native binaries it pulls in; pinning `fern-api`'s own version does - not pin what its dependencies resolve to on each run, since npm re-resolves - the whole tree from semver ranges every time. A guard now asserts every - workflow's `npm install` carries `--ignore-scripts`. The developer-facing - install instructions (`docs/README.md`, `docs/AGENTS.md`, - `docs/developer-guide.mdx`) carry the same two flags now, so a contributor - or agent following them locally gets the same hardening CI does. +- The Fern CLI is now installed from a committed lockfile, `fern/package-lock.json`, + with `npm ci --prefix fern --ignore-scripts --omit=optional`, and both workflows + invoke `./fern/node_modules/.bin/fern` rather than a binary on `PATH`. Previously + `publish-docs.yml` ran `npm install`, which re-resolves every transitive + dependency from semver ranges on each run: pinning `fern-api`'s own version + pinned nothing beneath it, and `@scarf/scarf` — reached through `fern-api`'s + optional dependency on `@boundaryml/baml` — declares a `postinstall` script that + executed in the job holding `FERN_TOKEN`. The lockfile pins every package to an + exact version and integrity hash, `--ignore-scripts` stops lifecycle code + running, and `--omit=optional` (which npm honours for this local install, but + silently ignores for a `--global` one) leaves `fern-api` alone in the tree. + Guards now assert that no workflow resolves a Node dependency tree from the + registry, that every npm command carries `--ignore-scripts`, and that + `fern/package.json`, `fern/package-lock.json` and `fern/fern.config.json` all + name the same CLI version. The developer-facing install instructions + (`docs/README.md`, `docs/AGENTS.md`, `docs/developer-guide.mdx`) use the same + lockfile install, so a contributor or agent following them gets the tree CI runs. - The action-pinning guard now accepts a same-repo composite action or reusable workflow (`uses: ./...`) without requiring a commit SHA, since GitHub always resolves a local reference from the caller's own commit and diff --git a/docs/AGENTS.md b/docs/AGENTS.md index fee644b7..e26ab011 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -64,11 +64,12 @@ Navigation order, page titles, and slugs are defined in ## Verify before committing -Requires **Node.js 22+** and the Fern CLI (`npm install -g --ignore-scripts --omit=optional fern-api`). +Requires **Node.js 22+**. Install the pinned CLI from the committed lockfile with +`npm ci --prefix fern --ignore-scripts --omit=optional`. ```bash -fern check # validate docs.yml config and all links — must pass -fern docs dev # optional live preview at http://localhost:3000 +./fern/node_modules/.bin/fern check # validate docs.yml config and all links — must pass +./fern/node_modules/.bin/fern docs dev # optional live preview at http://localhost:3000 ``` `fern check` must pass; it is the same gate the site build relies on. Publishing diff --git a/docs/README.md b/docs/README.md index e8f210d0..4397e358 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,14 +29,14 @@ published automatically from `main` via the Fern GitHub integration. Prerequisites: Node.js 22+ and npm 10+ (the versions the Fern CLI requires). ```bash -# Install the Fern CLI -npm install -g --ignore-scripts --omit=optional fern-api +# Install the pinned Fern CLI from the committed lockfile +npm ci --prefix fern --ignore-scripts --omit=optional # From the repo root, preview the site with live reload -fern docs dev +./fern/node_modules/.bin/fern docs dev # Validate the docs configuration and links -fern check +./fern/node_modules/.bin/fern check ``` `fern docs dev` serves the site at and reloads on changes diff --git a/docs/developer-guide.mdx b/docs/developer-guide.mdx index 0dd56d71..a42ac291 100644 --- a/docs/developer-guide.mdx +++ b/docs/developer-guide.mdx @@ -83,17 +83,18 @@ The documentation site is built with [Fern](https://buildwithfern.com/) from the ### Install the Fern CLI - Requires Node.js 22+ and npm 10+. `--ignore-scripts --omit=optional` keeps npm from running any - package's install-time scripts, the same hardening CI applies to this install. + Requires Node.js 22+ and npm 10+. `fern/package-lock.json` pins the CLI and every package + below it; `npm ci` reproduces that tree exactly, and the flags keep npm from running any + install-time scripts. This is the same install CI performs. - ```bash title="Install fern-api" - npm install -g --ignore-scripts --omit=optional fern-api + ```bash title="Install the pinned fern-api" + npm ci --prefix fern --ignore-scripts --omit=optional ``` ### Preview with live reload ```bash title="Local docs preview" - fern docs dev + ./fern/node_modules/.bin/fern docs dev ``` The site serves at `http://localhost:3000` and reloads on changes to `.mdx` files or `fern/docs.yml`. @@ -101,7 +102,7 @@ The documentation site is built with [Fern](https://buildwithfern.com/) from the ### Validate before pushing ```bash title="Check config and links" - fern check + ./fern/node_modules/.bin/fern check ``` `fern check` must pass — it is the same gate the site build relies on.