From aa293c4076b9621fb7dd343cdafedb6a0e6c43ec Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 12:51:52 -0700 Subject: [PATCH 01/17] Host the Hardened Secret and Env Readers in the Hub and Carry Them as a Tree The hub now hosts lib/sh/secrets.sh, the secret and env family that a fleet repository runs on its own hosts: secret_state, the shape, content, and set verdicts, require_secret, and read_env_value. A repository that declares the new host-secrets type carries it byte-for-byte at lib/fleet/sh through a spec/files.json tree. The hub tests it, since a consumer's copy is the same bytes. repo_gate.py gains a carried-trees check. It compares each carried tree with the hub tree beside the gate, which in CI is the pinned action's, and reports a difference as a note and an Actions warning. A difference never fails the gate, so a hub fix does not break every downstream pull request. GOVERNANCE.md "Hub-Hosted Tooling" states why a runtime library is carried rather than reached. Closes #3505 Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/action.yml | 2 +- .github/actions/repo-gate/repo_gate.py | 101 ++++++++ AUDIT.md | 1 + GOVERNANCE.md | 3 + lib/sh/secrets.sh | 243 +++++++++++++++++++ reports/conformance-matrix.md | 1 + scripts/README.md | 1 + spec/files.json | 3 +- spec/project-types.json | 6 + spec/scope-model.md | 2 +- spec/type-model.md | 5 + tests/test_cli_usage.py | 1 + tests/test_repo_gate.py | 137 +++++++++++ tests/test_secrets_sh.py | 320 +++++++++++++++++++++++++ 14 files changed, 823 insertions(+), 3 deletions(-) create mode 100644 lib/sh/secrets.sh create mode 100755 tests/test_secrets_sh.py diff --git a/.github/actions/repo-gate/action.yml b/.github/actions/repo-gate/action.yml index 549717439..64372e25c 100644 --- a/.github/actions/repo-gate/action.yml +++ b/.github/actions/repo-gate/action.yml @@ -1,5 +1,5 @@ name: Repository gate -description: Check action pins, line-ending contracts, and composite action shell and schema. +description: Check action pins, line-ending contracts, composite action shell and schema, and carried tree currency. inputs: exclude-globs: diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 832bd76b2..9f5c0f48a 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -8,6 +8,7 @@ eol .editorconfig <-> .gitattributes disagreement 40 findings eol-coverage Git attribute resolution differs from policy count not recorded composite-actions shellcheck and schema coverage of action.yml count not recorded + carried-trees a manifest-owned tree differing from the hub's count not recorded `eol-coverage` asks Git how representative paths resolve. This proves the global text default reaches Python, shell, Dockerfiles, workflow YAML, and extensionless scripts without maintaining @@ -36,6 +37,7 @@ import argparse import json +import os import re import shutil import subprocess @@ -769,11 +771,110 @@ def check_composite_actions(root: Path, files: list[str]) -> list[str]: return hits +def gate_hub_root() -> Path | None: + """The hub tree this gate file sits in, or None where it was lifted out of one. + + The pinned action runs from the whole hub tree at its pin, and so does a run from a hub + checkout. A single-file fetch has no tree around it, and whatever happens to sit three levels + above the file is not the hub, so the path is trusted only where its own parents name it. + """ + here = Path(__file__).resolve() + if [part.name for part in here.parents[:3]] != ["repo-gate", "actions", ".github"]: + return None + root = here.parents[3] + return root if (root / "spec" / "files.json").is_file() else None + + +def workflow_command_data(text: str) -> str: + """`text` escaped as a workflow command's message, per the runner's own percent encoding.""" + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def hub_tree_files(source: Path) -> dict[str, bytes]: + """Every file under `source`, keyed by its path relative to it, never following a link.""" + found: dict[str, bytes] = {} + for current, _, filenames in os.walk(source, followlinks=False): + for name in filenames: + path = Path(current) / name + found[path.relative_to(source).as_posix()] = path.read_bytes() + return found + + +def check_carried_trees(root: Path, files: list[str]) -> list[str]: + """Report each manifest-owned tree the scanned repository carries that differs from the hub's. + + `spec/files.json` declares the trees a repository carries byte-for-byte, and a carried copy + goes stale silently at the next hub fix. Comparing against the hub tree this gate runs from + makes the answer reproducible at the action's pin, so a Dependabot bump of that pin is what + surfaces a refresh the audit and resync then apply. + + A difference is a note rather than a finding, so it never fails the gate. Failing would break + every downstream pull request on each hub fix, and a byte difference cannot tell a stale copy + from an edited one, which the audit tells apart and disposes of. Which trees a repository owes + is also the audit's, read from its declared types, so a tree absent here is not compared. + The target side is read from the tracked files alone, so an untracked file is not a difference. + """ + hub = gate_hub_root() + if hub is None: + NOTES.append( + "no hub tree is beside this gate, as with a single-file fetch, so no carried tree was compared." + ) + return [] + if hub == root.resolve(): + NOTES.append( + "the scanned repository is the hub, whose trees are the sources rather than copies." + ) + return [] + try: + trees = json.loads((hub / "spec" / "files.json").read_text(encoding="utf-8"))["trees"] + except (OSError, ValueError, KeyError) as error: + return [f"spec/files.json in the hub tree could not be read: {error}"] + carried = 0 + for tree in trees: + target = tree["target"].rstrip("/") + prefix = target + "/" + tracked_here = [path for path in files if path.startswith(prefix)] + if not tracked_here: + continue + carried += 1 + if tree["include"] != ["**/*"]: + NOTES.append( + f"{target} declares include patterns this check does not read, so it was not compared." + ) + continue + source = hub_tree_files(hub / tree["source"]) + copy: dict[str, bytes | None] = {} + for path in tracked_here: + on_disk = root / path + copy[path[len(prefix) :]] = ( + None if on_disk.is_symlink() or not on_disk.is_file() else on_disk.read_bytes() + ) + missing = sorted(set(source) - set(copy)) + extra = sorted(set(copy) - set(source)) + changed = sorted(path for path in set(source) & set(copy) if copy[path] != source[path]) + if missing or extra or changed: + # Shown on the pull request rather than only in the log, from the manifest's own paths, so nothing a scanned repository names reaches the command. + if os.environ.get("GITHUB_ACTIONS") == "true": + print( + "::warning title=carried-trees::" + + workflow_command_data(f"{target} differs from the hub's {tree['source']}") + ) + NOTES.append( + f"{target} differs from the hub's {tree['source']}: {len(changed)} changed, " + f"{len(missing)} missing, {len(extra)} extra, first " + f"{(changed + missing + extra)[0]}. Refresh it from the hub, or let the audit say " + "whether it is a stale copy or an edited one." + ) + NOTES.append(f"compared {carried} carried tree(s) against the hub tree beside this gate.") + return [] + + CHECKS = { "sha-pin": check_sha_pin, "eol": check_eol, "eol-coverage": check_eol_coverage, "composite-actions": check_composite_actions, + "carried-trees": check_carried_trees, } diff --git a/AUDIT.md b/AUDIT.md index 953a6290e..6679468a9 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -58,6 +58,7 @@ Otherwise read its `types[]`. If the entry is `classificationPending` (a backlog - `Dockerfile` + a docker build/push workflow -> `docker`, an `upstream-version.json` tracker -> `upstream-wrapper`. - `custom_components/*/manifest.json` + `hacs.json` -> `homeassistant`, a codegen workflow -> `codegen`, no `build-*` task -> `source-only`, governance-only -> `docs`. - `hugo.yaml` / `hugo.toml` / `config/_default/hugo.yaml` -> `hugo`. A repo may carry it alongside `source-only`, since a site deploy leaf is not a `build-*` task and both declarations stay true. +- a carried `lib/fleet/sh/secrets.sh`, or a shell script the repo runs that sources or reads a secret or env file from disk -> `host-secrets`, which selects that carried tree. ## 3. Applicability Gate diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 923ef5f04..922bc15f4 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -139,6 +139,8 @@ The fleet's tooling lives in the hub once and a repository runs it from there ra **What a repository carries and what it reaches is decided by what the content is.** It carries the content it is audited against and the configuration that describes it, meaning its rule text and the files the manifest declares. It reaches machinery whose content is identical in every repository, meaning the prose and repository gates, the review digest, and the configuration script, because a file holding no per-repo content is a copy whose only future is to go stale. A tool named in a carried rule is therefore named as the hub's, since the alternative is a pointer to a path the reader does not have, and a pointer that resolves nowhere teaches the reader that a pointer in carried text is decorative. +**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root, so fetching it from the hub at run time lets a GitHub outage or a bad `main` commit stop a backup, and it runs code the repository never reviewed against its own secrets. Offline operation and review in the repository that runs it outweigh the staleness of a copy there, which is not true of a gate, where "the tool did not run" is an acceptable answer and CI is the backstop. The hub hosts the library once, and a repository whose declared types select it carries it byte-for-byte as a tree the hub's `spec/files.json` declares. The repository extends it in a file of its own that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds and reports a difference as a warning that never fails it, since a failure would break every downstream pull request on each hub fix, and the audit and the resync apply the refresh. A local hook's `hub_fetch_run.py` is carried by the same reasoning, since it is the loader that reaches the hub and so cannot itself be reached. + **Reaching it is a checkout of the hub rather than a copy of one file.** A tool reads the payloads, tables, and sibling modules beside it, so a single file lifted out of the tree runs against whatever the caller happens to have, which is the copy problem again in a shorter loop. Read `main`, the promoted and gated state, and fetch immediately before running, because a clone is whatever it last fetched rather than the branch it names, and a stale clone answers confidently instead of failing. Name the tool by its path in that checkout and name the target explicitly, since a tool that defaults to the current directory or the current repository resolves somewhere either way, and a result computed against the wrong repository is well-formed. Which directory the command runs in is the tool's own contract rather than a rule here, so a gate reading a working tree runs in the repository under test while a tool taking its target as an argument runs anywhere and is given one. What the rule forbids is letting a default decide which repository the answer is about. **A loader is outside this section rather than exempt from it.** The rule above governs a tool that reads hub content, because a tool reads the payloads, tables, and sibling modules beside it. A loader reads none of them: it obtains a tree and hands control to a tool inside that tree, on a host that cannot yet obtain one. The bound is what it may contain rather than who runs it, and it is one line: a loader references no path inside the tree it fetches except the single entry point it hands control to, and everything else it touches is the machine or the network. A loader that grows a second path into that tree has become a tool and is governed above. @@ -374,6 +376,7 @@ Every repo's GitHub repository details (the About panel) follow a fixed conventi - [`.github/skills/`](./.github/skills/): generated in the hub from `.agents/skills/` and carried verbatim to every repo, so a fix to a Skill belongs at that source rather than in any copy of this tree. - [`.claude-plugin/fleet-skills/`](./.claude-plugin/fleet-skills/): hub-only, the same Skills as a Claude plugin. The `marketplace.json` beside it names the plugin rather than the Skills, is hand-authored, and `build_dist.py` never writes it. - [`scripts/`](./scripts/): the fleet's lint and review tooling, with the deterministic checks also gating CI, for the documented rules no CI linter checks. Hosted here and reached rather than carried, per "Hub-Hosted Tooling". +- [`lib/`](./lib/): runtime libraries the hub hosts and a repository carries byte-for-byte, per "Hub-Hosted Tooling", each under `lib/fleet/` in the repository whose declared types select it. `lib/sh/` is the shell secret and env readers, carried to a `host-secrets` repository. - [`tests/`](./tests/): hub-only, the unit tests for the hub's own tooling, which the shared validator runs under coverage as the hub's declared Python directory. - [`docs/`](./docs/): hub-only procedure and measurement docs (host setup, repository configuration, signing, agent token cost), none of them carried downstream. - [`host-setup/`](./host-setup/): hub-only, the installers that bring a machine up to the host contract `docs/host-setup.md` states, one tree each for Linux and Windows, with the cross-platform agent-safety install beside them. diff --git a/lib/sh/secrets.sh b/lib/sh/secrets.sh new file mode 100644 index 000000000..3fb9c3d09 --- /dev/null +++ b/lib/sh/secrets.sh @@ -0,0 +1,243 @@ +#!/bin/bash + +# Meant to be sourced, never run: `. "$REPO/lib/fleet/sh/secrets.sh"` from a script that has already found its own tree through `BASH_SOURCE`. +# It sets no shell option, since sourcing it would change the caller's own, and every function answers through its output or its status under any of them. +# +# The hub hosts this file and a repository carries it byte-for-byte, per GOVERNANCE.md "Hub-Hosted Tooling", so a repository extends it in a file of its own that sources it, never by editing it. +# A caller that needs these functions in an elevated shell ships them there by name with `declare -f`, since sourcing this file in the calling shell does not reach one started by `sudo`. +# A function here that calls another is shipped with it, so `secret_verdict` needs `secret_shape_verdict`, `secret_state`, `unresolved_link_reason`, `secret_content_verdict`, `secret_content_state`, and `secret_content_refusal`. + +# `unresolved_link_reason` +# Echoes the clause that says why a symlink answering `-L` true and `-e` false does not resolve, worded for the account running it, so every site that reports such a link composes one sentence instead of spelling its own. +# Root with its DAC override enters every directory, so a link it cannot resolve has a target that is missing or loops back on itself, and the clause says only that. +# A uid 0 without the override, such as one in a user namespace facing an unmapped owner, is read as root all the same. +# Any other account cannot tell those two from a target sitting behind a directory it cannot enter, and the clause names all three. +# A caller that tells the two wordings apart does so by the ending ` sits behind a directory this account cannot enter`, fixed text here. +unresolved_link_reason() { + if [[ $EUID -eq 0 ]]; then + echo "its target is missing or loops back on itself" + else + echo "its target is missing, loops back on itself, or sits behind a directory this account cannot enter" + fi +} + +# `secret_state` +# One word for what a secret file is, because a gate that skips on a bare `-f` reads four unrelated states as one, since `-f` answers false for each of them. +# `absent` only where nothing is at the path and every directory leading to it is one this account can actually enter, which is the single state a gate may skip on. +# `damaged ` where the path or a directory leading to it is a symlink that does not resolve, or is present in a shape a secret cannot be read from. +# `unknown ` where a directory leading to it cannot be entered, or the path is not absolute, so the path's absence was never established rather than observed. +# `present` where a regular file is there. +# The walk starts at the root rather than at the immediate parent, since a single unenterable ancestor makes every component below it answer false to `-e` and that would otherwise read as a proven absence. +# See `secret_verdict` below for the full check built on it. +secret_state() { + local path="$1" walked="" part index last + if [[ $path != /* ]]; then + echo "unknown $path is not an absolute path" + return 0 + fi + # A doubled or trailing slash names the same component again, so both are removed before the split rather than becoming empty components the walk would have to skip. + while [[ $path == *//* ]]; do + path="${path//\/\//\/}" + done + path="${path%/}" + # The trim above empties a path that was only slashes, which would otherwise walk no components at all and fall through to `present` without ever reaching the leaf test. + # The root is a directory, so it gets the same answer any other non-regular-file leaf gets rather than a fifth state. + if [[ -z $path ]]; then + echo "damaged / is not a regular file" + return 0 + fi + local -a parts + # Split on `/` alone, since `read` stops at the first newline and a path component may hold one, and a split that fails answers rather than walking no components. + readarray -d / -t parts < <(printf '%s' "${path#/}") || { + echo "unknown $path could not be split into its components" + return 0 + } + last=$((${#parts[@]} - 1)) + for index in "${!parts[@]}"; do + part="${parts[index]}" + walked="$walked/$part" + # `-L` before `-e` throughout: a symlink that does not resolve answers false to `-e`, which is exactly the state a bare existence test reads as an absence. + # Reported as unresolved rather than as a missing target, since `-e` cannot separate a target that is gone from one that loops back on itself or, for a non-root account, one sitting behind a directory this account cannot enter, and asserting any one of them would claim more than the test established. + if [[ -L $walked && ! -e $walked ]]; then + echo "damaged $walked is a symlink that does not resolve, so $(unresolved_link_reason)" + return 0 + fi + if [[ ! -e $walked ]]; then + echo absent + return 0 + fi + if [[ $index -eq $last ]]; then + if [[ ! -f $walked ]]; then + echo "damaged $walked is not a regular file" + return 0 + fi + echo present + return 0 + fi + if [[ ! -d $walked ]]; then + echo "damaged $walked is not a directory, so it can hold no secret" + return 0 + fi + if [[ ! -x $walked ]]; then + echo "unknown $walked cannot be entered by this account" + return 0 + fi + done + echo present +} + +# `secret_shape_verdict` [private|shared] +# What `secret_verdict` below answers, from the file's metadata alone, without opening it. +# `readable` where it is a regular file, not a symlink, gives group and other no access, and this account can read it. +# `shared` drops the group and other check alone, for a secret a container reads as a Docker secret, which stays group and other readable by design. +# `absent` where nothing is there, which `secret_state` above establishes rather than infers, so a caller whose secret is optional may skip on it. +# `refused ` for everything else, the sentence naming the path and why, so a caller prints it as the reason rather than composing its own. +# A symlink is refused by name even where it resolves to a private file, since `stat -c %a` reads the link's own 777 rather than its target's mode, and a link is also a second place the file's content can be repointed from. +# A caller running as root can call this one alone and run `secret_content_state` in the shell it reads the file with, so root never opens the file. +secret_shape_verdict() { + local path="$1" access="${2:-private}" state mode + # Refused rather than normalized, since the caller reads the spelling it was given, and the kernel answers a trailing slash on a file with "Not a directory". + if [[ $path == */ ]]; then + printf 'refused %s ends in a slash, so it names a directory rather than a file\n' "$path" + return 0 + fi + state=$(secret_state "$path") + case $state in + present) ;; + absent) + echo absent + return 0 + ;; + *) + printf 'refused %s\n' "${state#* }" + return 0 + ;; + esac + if [[ -L $path ]]; then + printf 'refused %s is a symlink, and must be the file itself\n' "$path" + return 0 + fi + # Validated as octal digits before `(( ))` reads it, since a non-octal value there fails the test the same way a private mode passes it. + if ! mode=$(stat -c '%a' -- "$path") || [[ ! $mode =~ ^[0-7]{1,4}$ ]]; then + printf 'refused the mode of %s could not be read, so whether it gives group or other access cannot be established\n' "$path" + return 0 + fi + if [[ $access != shared ]] && ((8#$mode & 8#077)); then + printf 'refused %s is mode %s, and must give group and other no access\n' "$path" "$mode" + return 0 + fi + if [[ ! -r $path ]]; then + printf 'refused %s is mode %s and owned by %s, so this account cannot read it\n' "$path" "$mode" "$(stat -c '%U' -- "$path")" + return 0 + fi + echo readable +} + +# `secret_content_state` +# `clean` where the file `secret_shape_verdict` passed holds no carriage return, `crlf` where it holds one, and `unchecked` where the check could not run. +# A FIFO or device in place of the file, or a symlink to one, answers `unchecked`, since `-D skip` passes such a file unread. +# Refused rather than left to each reader, since each reader fails on a carriage return differently. +# A sourcing shell under errexit stops on a blank CRLF line, and one without it keeps the carriage return in every value. +secret_content_state() { + # `grep` streams the file rather than holding it, and `-D skip` keeps `grep` itself from blocking on a FIFO swapped in after the shape checks. + local found=0 + LC_ALL=C grep -qF -D skip -- $'\r' "$1" || found=$? + case $found in + 0) echo crlf ;; + 1) + if [[ -f $1 ]]; then + echo clean + else + echo unchecked + fi + ;; + *) echo unchecked ;; + esac +} + +# `secret_content_refusal` +# The `refused ` for a `secret_content_state` other than `clean`, built without opening the file, and any state but `crlf` reads as `unchecked`. +secret_content_refusal() { + if [[ $2 == crlf ]]; then + printf 'refused %s holds a carriage return, most often from CRLF line endings, and must use LF line endings only\n' "$1" + else + printf 'refused the check of %s for a carriage return could not run, so whether it holds one cannot be established\n' "$1" + fi +} + +# `secret_content_verdict` +# `readable` where `secret_content_state` answers `clean`, and its `secret_content_refusal` otherwise. +secret_content_verdict() { + local state + state=$(secret_content_state "$1") + if [[ $state == clean ]]; then + echo readable + else + secret_content_refusal "$1" "$state" + fi +} + +# `secret_verdict` [private|shared] +# One word for whether may be read as a secret: `secret_shape_verdict`'s answer, then `secret_content_verdict`'s where the shape is `readable`. +# A reader in another language runs this function rather than restating its checks, so the rule exists once, here. +secret_verdict() { + local verdict + verdict=$(secret_shape_verdict "$1" "${2:-private}") + if [[ $verdict != readable ]]; then + printf '%s\n' "$verdict" + return 0 + fi + secret_content_verdict "$1" +} + +# `secret_set_verdict` ... +# One answer for a set of secrets read all or none, from `secret_verdict` on each path. +# `readable` where every path is, which an empty list is too. +# `missing ` where any is absent, naming the final component of each absent path in the order given, separated by single spaces. +# `refused ` otherwise, the first refusal in the order given, so the sentence names the first file a reader of the set would stop on. +# A missing file outranks a refusal, since the set cannot be complete whatever the refused file turns out to be. +# Naming the refusal first would send the operator to fix a mode, only for the next run to name the files still missing. +secret_set_verdict() { + local path verdict missing="" refused="" + for path in "$@"; do + verdict=$(secret_verdict "$path") + case $verdict in + readable) ;; + absent) missing="$missing ${path##*/}" ;; + *) [[ -n $refused ]] || refused="$verdict" ;; + esac + done + if [[ -n $missing ]]; then + printf 'missing%s\n' "$missing" + elif [[ -n $refused ]]; then + printf '%s\n' "$refused" + else + echo readable + fi +} + +# `require_secret` +# Succeeds only where `secret_verdict` answers `readable`, and otherwise prints why to stderr and fails, so a secret that must be there is one bare call before the line that sources it. +# A bare call under `set -e` stops the script on that failure, where a call in an `if` or before `||` would not. +require_secret() { + local verdict + verdict=$(secret_verdict "$1") + case $verdict in + readable) return 0 ;; + absent) printf 'ERROR: %s is not there\n' "$1" >&2 ;; + *) printf 'ERROR: %s\n' "${verdict#refused }" >&2 ;; + esac + return 1 +} + +# `read_env_value` +# Prints as the env file named by sets it, and fails where any line of that file fails. +# A bash of its own rather than a subshell, because a caller reading this inside an `if` condition has errexit off in every subshell as well, so only the last line of the file would count. +# The child also keeps `set -a` from exporting that file's credentials into anything this script spawns, and `unset` keeps an operator's already-sourced shell from standing in for the file. +# The name is held in a variable of the child's own rather than read from `$2` at the end, since the file runs with the positional parameters in view and can change them. +# What the file itself prints goes to /dev/null rather than into the value, and `BASH_ENV`, `POSIXLY_CORRECT` and `SHELLOPTS` are dropped so nothing but the file runs ahead of the read. +# It checks nothing about the file, so a caller runs `require_secret` on it first. +read_env_value() { + # shellcheck disable=SC2016 # Every expansion belongs to the child bash. + env -u BASH_ENV -u POSIXLY_CORRECT -u SHELLOPTS bash -Eeuo pipefail -c 'env_read_name=$2; unset -v "$env_read_name"; set -a; source "$1" >/dev/null; set +a; printf "%s" "${!env_read_name:-}"' _ "$1" "$2" +} diff --git a/reports/conformance-matrix.md b/reports/conformance-matrix.md index f02bf30b2..7af653fe9 100644 --- a/reports/conformance-matrix.md +++ b/reports/conformance-matrix.md @@ -20,6 +20,7 @@ The primary shapes are stood up as whole repos. The **composable targets** (`nug | `upstream-wrapper` | - | not-tested | - | Tag from a committed state file, not SemVer2. | | `codegen` | - | not-tested | - | Deterministic matrix over both branches. | | `docs` | ProjectTemplate | not-tested | - | Governance hub, and CI is lint-only. | +| `host-secrets` overlay | - | not-tested | - | Layered onto a base shape, and carries the hub's `lib/sh` secret and env readers at `lib/fleet/sh` byte-for-byte. HomeAutomation-Config is the first adopter, by its own adoption issue. | | `operational` config | - | not-tested | - | `workflowModel: operational`, direct signed commits to `develop`, promotion-PR gate. HomeAutomation-Config was the reference until its 2026-09-24 reclassification to `release`, and no remaining operational repository (ESPHome-Config, HomeAssistant-Config, Vantage-Config) has taken its place yet. | ## Composable Targets diff --git a/scripts/README.md b/scripts/README.md index b3fa46b04..8f8111012 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,6 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. +- `carried-trees`: each tree [`spec/files.json`][files] declares that the scanned repository tracks at its target is compared byte-for-byte with the hub tree beside the gate, the pinned action's tree in CI and the checkout in a local run. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions, and never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit gives each its disposition and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a hub `main` newer than the pin also warns until Dependabot advances it. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it and says so in a note, and the hub scanning itself compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/spec/files.json b/spec/files.json index b5a6d7b8e..e29a3b69a 100644 --- a/spec/files.json +++ b/spec/files.json @@ -2,7 +2,8 @@ "$schema": "./files.schema.json", "note": "The standardization baseline: files, sections, and fully owned trees a fleet repo is expected to carry. See spec/scope-model.md for applicability and spec/fidelity-model.md for fidelity.", "trees": [ - { "source": ".github/skills", "target": ".github/skills", "fidelity": "verbatim-tree", "appliesTo": "*", "include": ["**/*"], "prune": true } + { "source": ".github/skills", "target": ".github/skills", "fidelity": "verbatim-tree", "appliesTo": "*", "include": ["**/*"], "prune": true }, + { "source": "lib/sh", "target": "lib/fleet/sh", "fidelity": "verbatim-tree", "appliesTo": ["host-secrets"], "include": ["**/*"], "prune": true } ], "baseline": [ { "path": "CLAUDE.md", "fidelity": "verbatim", "whole": true, "appliesTo": "*" }, diff --git a/spec/project-types.json b/spec/project-types.json index a593b3008..f468208d7 100644 --- a/spec/project-types.json +++ b/spec/project-types.json @@ -130,6 +130,12 @@ "checks": [ { "id": "docs.ci.lintonly", "verdict": "intent", "assert": "CI runs linting only (markdownlint, cspell, JSON schema, actionlint); no build/test.", "intentRef": "AGENTS.md" } ] + }, + "host-secrets": { + "detect": ["lib/fleet/sh/secrets.sh", "a shell script the repo runs that sources or reads a secret or env file from disk"], + "checks": [ + { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a secret or env file sources the carried lib/fleet/sh/secrets.sh and checks the file with require_secret, secret_verdict, or secret_set_verdict before reading it, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file that sources the library, never in an edit to the carried copy, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#hub-hosted-tooling" } + ] } }, "crossCutting": { diff --git a/spec/scope-model.md b/spec/scope-model.md index 62e7bd057..dcde572ae 100644 --- a/spec/scope-model.md +++ b/spec/scope-model.md @@ -15,7 +15,7 @@ A selector is one token from one of four **disjoint** namespaces. Because the na | Namespace | Tokens | Source of truth | | --- | --- | --- | -| project type | `csharp` `nuget` `pypi` `python` `cpp` `dotnet-publish` `docker` `homeassistant` `eda` `codegen` `upstream-wrapper` `source-only` `hugo` `docs` | [`spec/project-types.json`][project-types] | +| project type | `csharp` `nuget` `pypi` `python` `cpp` `dotnet-publish` `docker` `homeassistant` `eda` `codegen` `upstream-wrapper` `source-only` `hugo` `docs` `host-secrets` | [`spec/project-types.json`][project-types] | | workflow model | `release` `operational` | [`registry/repos.schema.json`][repos-schema] | | release trigger | `two-phase` `dispatch-only` `none` | [`registry/repos.schema.json`][repos-schema] | | consumer model | `push` `pull` | [`registry/repos.schema.json`][repos-schema] | diff --git a/spec/type-model.md b/spec/type-model.md index d2adf7ccb..bf2663ba7 100644 --- a/spec/type-model.md +++ b/spec/type-model.md @@ -56,6 +56,10 @@ There is no `static-site` to `hugo` hierarchy while the type has one member. Ins A generator type declares no `profiles`. Build versus lint-only is a depth of *language* presence, so a profile on a generator type would assert nothing, and `spec/validate.py` rejects a declared profile whose type does not define one. +## Runtime Libraries + +A **runtime library** type declares that the repo's own runtime executes a library the hub hosts, so the type's token is what selects the library's tree in [`spec/files.json`][files] and the repo carries it byte-for-byte. `host-secrets` is the one such type, for a repo whose scripts read secret and env files on a host it owns, and it carries `lib/sh` from the hub to `lib/fleet/sh`. The type is declared rather than inferred like any other, since a repo that reads one secret inline and one that runs a fleet of host services look alike to a file scan. A runtime library type declares no `profiles`, for the reason a generator type declares none. + ## Changing the type set carries review weight The set of types, their profiles, and each type's checks is governed, like the section and fidelity models. @@ -72,6 +76,7 @@ The set of types, their profiles, and each type's checks is governed, like the s [audit]: ./audit.py [fidelity-model]: ./fidelity-model.md +[files]: ./files.json [repos]: ../registry/repos.json [schema]: ./project-types.schema.json [section-model]: ./section-model.md diff --git a/tests/test_cli_usage.py b/tests/test_cli_usage.py index f04f7b400..216491d28 100755 --- a/tests/test_cli_usage.py +++ b/tests/test_cli_usage.py @@ -69,6 +69,7 @@ "host-setup/agent-safety/claude/stray-process-sweep.py": "a Claude Code hook", "host-setup/agent-safety/claude/tool-containment.py": "a Claude Code hook", "host-setup/agent-safety/claude/test_install.py": "a test module", + "lib/sh/secrets.sh": "a library a script sources", } BEFORE_FLAG = { diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 451dce762..3ea2e4247 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -14,6 +14,7 @@ import functools import importlib import io +import json import os import re import shutil @@ -1337,6 +1338,142 @@ def test_a_schema_violation_fails(self) -> None: self.assertTrue(any(h.startswith("schema:") for h in hits)) +class TestCarriedTrees(TreeCase): + """A constructed hub beside the gate and a constructed copy, so each case proves one difference.""" + + def setUp(self) -> None: + super().setUp() + self.hub = self.tmp / "hub" + self.repo = self.tmp / "repo" + self.manifest([{"source": "lib/sh", "target": "lib/fleet/sh", "include": ["**/*"]}]) + self.write(self.hub / "lib/sh/secrets.sh", "hub\n") + self.enterContext(mock.patch.object(repo_gate, "gate_hub_root", return_value=self.hub)) + + def manifest(self, trees: list[dict[str, object]]) -> None: + self.write(self.hub / "spec/files.json", json.dumps({"trees": trees})) + + def write(self, path: Path, content: str) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path.relative_to(self.repo).as_posix() if path.is_relative_to(self.repo) else "" + + def run_check(self, files: list[str], actions: bool = False) -> tuple[list[str], str]: + with mock.patch.dict(os.environ), contextlib.redirect_stdout(io.StringIO()) as out: + if actions: + os.environ["GITHUB_ACTIONS"] = "true" + else: + os.environ.pop("GITHUB_ACTIONS", None) + hits = repo_gate.check_carried_trees(self.repo, files) + return hits, out.getvalue() + + def test_an_identical_copy_raises_no_difference(self) -> None: + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] + self.assertEqual(self.run_check(files), ([], "")) + self.assertEqual( + repo_gate.NOTES, ["compared 1 carried tree(s) against the hub tree beside this gate."] + ) + + def test_a_changed_byte_is_a_note_and_never_a_finding(self) -> None: + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")] + hits, _ = self.run_check(files) + self.assertEqual(hits, []) + self.assertIn( + "lib/fleet/sh differs from the hub's lib/sh: 1 changed, 0 missing, 0 extra, first secrets.sh.", + repo_gate.NOTES[0], + ) + + def test_a_missing_and_an_extra_file_are_counted(self) -> None: + files = [self.write(self.repo / "lib/fleet/sh/local.sh", "mine\n")] + self.run_check(files) + self.assertIn("0 changed, 1 missing, 1 extra, first secrets.sh.", repo_gate.NOTES[0]) + + def test_an_untracked_file_is_not_a_difference(self) -> None: + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] + self.write(self.repo / "lib/fleet/sh/scratch.sh", "untracked\n") + self.run_check(files) + self.assertEqual(len(repo_gate.NOTES), 1) + + def test_a_link_in_the_copy_is_a_change(self) -> None: + (self.repo / "lib/fleet/sh").mkdir(parents=True) + (self.repo / "lib/fleet/sh/secrets.sh").symlink_to(self.hub / "lib/sh/secrets.sh") + self.run_check(["lib/fleet/sh/secrets.sh"]) + self.assertIn("1 changed", repo_gate.NOTES[0]) + + def test_a_tree_the_repository_does_not_carry_is_not_compared(self) -> None: + self.run_check([self.write(self.repo / "README.md", "")]) + self.assertEqual( + repo_gate.NOTES, ["compared 0 carried tree(s) against the hub tree beside this gate."] + ) + + def test_a_narrowed_include_is_named_rather_than_compared(self) -> None: + self.manifest([{"source": "lib/sh", "target": "lib/fleet/sh", "include": ["*.sh"]}]) + self.run_check([self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")]) + self.assertEqual( + repo_gate.NOTES[0], + "lib/fleet/sh declares include patterns this check does not read, so it was not compared.", + ) + + def test_an_unreadable_manifest_is_a_finding(self) -> None: + self.write(self.hub / "spec/files.json", "{") + hits, _ = self.run_check([]) + self.assertEqual(len(hits), 1) + self.assertTrue(hits[0].startswith("spec/files.json in the hub tree could not be read")) + + def test_actions_shows_the_difference_on_the_pull_request(self) -> None: + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")] + _, printed = self.run_check(files, actions=True) + self.assertEqual( + printed, "::warning title=carried-trees::lib/fleet/sh differs from the hub's lib/sh\n" + ) + + def test_the_annotation_carries_no_name_from_the_scanned_repository(self) -> None: + files = [self.write(self.repo / "lib/fleet/sh/x%0A::error::forged", "")] + _, printed = self.run_check(files, actions=True) + self.assertEqual(printed.count("\n"), 1) + self.assertNotIn("forged", printed) + + def test_outside_actions_nothing_is_printed(self) -> None: + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")] + self.assertEqual(self.run_check(files)[1], "") + + def test_the_hub_scanning_itself_compares_nothing(self) -> None: + self.assertEqual(repo_gate.check_carried_trees(self.hub, []), []) + self.assertEqual( + repo_gate.NOTES, + ["the scanned repository is the hub, whose trees are the sources rather than copies."], + ) + + +class TestGateHubRoot(TreeCase): + def place(self, relative: str) -> str: + path = self.tmp / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("", encoding="utf-8") + return str(path) + + def test_the_gate_in_its_hub_tree_finds_the_hub(self) -> None: + self.place("spec/files.json") + gate = self.place(".github/actions/repo-gate/repo_gate.py") + with mock.patch.object(repo_gate, "__file__", gate): + self.assertEqual(repo_gate.gate_hub_root(), self.tmp.resolve()) + + def test_a_lifted_gate_finds_none(self) -> None: + self.place("spec/files.json") + gate = self.place("fetched/repo_gate.py") + with mock.patch.object(repo_gate, "__file__", gate): + self.assertIsNone(repo_gate.gate_hub_root()) + + def test_the_layout_without_a_manifest_finds_none(self) -> None: + gate = self.place(".github/actions/repo-gate/repo_gate.py") + with mock.patch.object(repo_gate, "__file__", gate): + self.assertIsNone(repo_gate.gate_hub_root()) + + def test_a_lifted_gate_reports_that_it_compared_nothing(self) -> None: + with mock.patch.object(repo_gate, "gate_hub_root", return_value=None): + self.assertEqual(repo_gate.check_carried_trees(self.tmp, []), []) + self.assertIn("no hub tree is beside this gate", repo_gate.NOTES[0]) + + class TestExcludeGlobs(unittest.TestCase): """A caller vendoring a subtree it does not author needs `tracked()` narrowed, not re-derived. diff --git a/tests/test_secrets_sh.py b/tests/test_secrets_sh.py new file mode 100755 index 000000000..fd2bab838 --- /dev/null +++ b/tests/test_secrets_sh.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Drive lib/sh/secrets.sh's verdicts and env read against constructed files. + +Each case builds its own tree under a temp root and asks the sourced library about it through a +child bash, so a case proves the function's answer for that shape rather than the state of this +host. A consumer carries the library byte-for-byte, so these cases are the only place it is tested. + +Run as `python3 tests/test_secrets_sh.py`, or under `python3 -m unittest discover -s tests`. +""" + +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +LIBRARY = ROOT / "lib" / "sh" / "secrets.sh" +TIMEOUT = 30 +ROOT_ACCOUNT = os.geteuid() == 0 + + +def run( + script: str, *args: str, env: dict[str, str] | None = None +) -> subprocess.CompletedProcess[str]: + """Source the library under the fleet header and run `script`, with `args` as `$1`...""" + return subprocess.run( + ["bash", "-c", f'set -Eeuo pipefail; . "$0"; {script}', str(LIBRARY), *args], + capture_output=True, + encoding="utf-8", + timeout=TIMEOUT, + env=env, + check=False, + ) + + +def answer(function: str, *args: str) -> str: + result = run(f'{function} "$@"', *args) + if result.returncode != 0: + raise AssertionError(f"{function} exited {result.returncode}: {result.stderr}") + return result.stdout.rstrip("\n") + + +def write(path: Path, content: str, mode: int = 0o600) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content.encode()) + path.chmod(mode) + return path + + +class Tree(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + # A temp root can itself sit below a symlink, such as /tmp on some hosts, and the walk reports a component by the spelling it was given. + self.root = self.root.resolve() + + def tearDown(self) -> None: + for current, dirnames, _ in os.walk(self.root): + for name in dirnames: + path = Path(current) / name + if not path.is_symlink(): + path.chmod(0o700) + self.temp.cleanup() + + +class SecretState(Tree): + def test_absent_where_nothing_is_there(self) -> None: + self.assertEqual(answer("secret_state", str(self.root / "missing")), "absent") + + def test_absent_below_a_missing_directory(self) -> None: + self.assertEqual(answer("secret_state", str(self.root / "gone" / "key")), "absent") + + def test_present_for_a_regular_file(self) -> None: + key = write(self.root / "key", "value\n") + self.assertEqual(answer("secret_state", str(key)), "present") + + def test_doubled_and_trailing_slashes_name_the_same_file(self) -> None: + write(self.root / "dir" / "key", "value\n") + self.assertEqual(answer("secret_state", f"{self.root}//dir///key"), "present") + + def test_a_relative_path_is_unknown(self) -> None: + self.assertEqual( + answer("secret_state", "relative/key"), "unknown relative/key is not an absolute path" + ) + + def test_the_root_is_not_a_regular_file(self) -> None: + self.assertEqual(answer("secret_state", "///"), "damaged / is not a regular file") + + def test_a_directory_leaf_is_damaged(self) -> None: + (self.root / "dir").mkdir() + self.assertEqual( + answer("secret_state", str(self.root / "dir")), + f"damaged {self.root}/dir is not a regular file", + ) + + def test_a_file_where_a_directory_belongs_is_damaged(self) -> None: + write(self.root / "file", "") + self.assertEqual( + answer("secret_state", str(self.root / "file" / "key")), + f"damaged {self.root}/file is not a directory, so it can hold no secret", + ) + + def test_a_dangling_leaf_link_is_damaged_rather_than_absent(self) -> None: + (self.root / "key").symlink_to(self.root / "nowhere") + self.assertTrue( + answer("secret_state", str(self.root / "key")).startswith( + f"damaged {self.root}/key is a symlink that does not resolve, so its target is missing" + ) + ) + + def test_a_dangling_parent_link_is_named(self) -> None: + (self.root / "dir").symlink_to(self.root / "nowhere") + self.assertTrue( + answer("secret_state", str(self.root / "dir" / "key")).startswith( + f"damaged {self.root}/dir is a symlink that does not resolve" + ) + ) + + def test_a_fifo_is_not_a_regular_file(self) -> None: + os.mkfifo(self.root / "key") + self.assertEqual( + answer("secret_state", str(self.root / "key")), + f"damaged {self.root}/key is not a regular file", + ) + + @unittest.skipIf(ROOT_ACCOUNT, "root enters a directory whatever its mode") + def test_an_unenterable_directory_is_unknown_rather_than_absent(self) -> None: + locked = self.root / "locked" + locked.mkdir() + locked.chmod(0o600) + self.assertEqual( + answer("secret_state", str(locked / "key")), + f"unknown {locked} cannot be entered by this account", + ) + + @unittest.skipIf(ROOT_ACCOUNT, "the root wording drops the unenterable-directory clause") + def test_the_link_reason_names_an_unenterable_directory_for_a_non_root_account(self) -> None: + self.assertTrue( + answer("unresolved_link_reason").endswith( + " sits behind a directory this account cannot enter" + ) + ) + + +class SecretVerdict(Tree): + def test_a_private_clean_file_is_readable(self) -> None: + key = write(self.root / "key", "value\n") + self.assertEqual(answer("secret_verdict", str(key)), "readable") + + def test_absent_passes_through(self) -> None: + self.assertEqual(answer("secret_verdict", str(self.root / "key")), "absent") + + def test_group_access_is_refused_by_default(self) -> None: + key = write(self.root / "key", "value\n", 0o640) + self.assertEqual( + answer("secret_verdict", str(key)), + f"refused {key} is mode 640, and must give group and other no access", + ) + + def test_shared_allows_group_and_other_read(self) -> None: + key = write(self.root / "key", "value\n", 0o644) + self.assertEqual(answer("secret_verdict", str(key), "shared"), "readable") + + def test_a_link_to_a_private_file_is_refused(self) -> None: + target = write(self.root / "real", "value\n") + (self.root / "key").symlink_to(target) + self.assertEqual( + answer("secret_verdict", str(self.root / "key")), + f"refused {self.root}/key is a symlink, and must be the file itself", + ) + + def test_a_trailing_slash_is_refused_rather_than_normalized(self) -> None: + key = write(self.root / "key", "value\n") + self.assertEqual( + answer("secret_verdict", f"{key}/"), + f"refused {key}/ ends in a slash, so it names a directory rather than a file", + ) + + def test_a_carriage_return_is_refused(self) -> None: + key = write(self.root / "key", "NAME=value\r\n") + self.assertEqual( + answer("secret_verdict", str(key)), + f"refused {key} holds a carriage return, most often from CRLF line endings, " + "and must use LF line endings only", + ) + + def test_a_damaged_state_becomes_a_refusal(self) -> None: + (self.root / "dir").mkdir() + self.assertEqual( + answer("secret_verdict", str(self.root / "dir")), + f"refused {self.root}/dir is not a regular file", + ) + + @unittest.skipIf(ROOT_ACCOUNT, "root reads a file whatever its mode") + def test_an_unreadable_file_is_refused_with_its_owner(self) -> None: + key = write(self.root / "key", "value\n", 0o000) + self.assertTrue( + answer("secret_verdict", str(key)).startswith(f"refused {key} is mode 0 and owned by ") + ) + + def test_a_fifo_content_check_is_unchecked(self) -> None: + os.mkfifo(self.root / "fifo") + self.assertEqual(answer("secret_content_state", str(self.root / "fifo")), "unchecked") + + +class SecretSetVerdict(Tree): + def test_an_empty_set_is_readable(self) -> None: + self.assertEqual(answer("secret_set_verdict"), "readable") + + def test_every_readable_file_makes_the_set_readable(self) -> None: + one = write(self.root / "one", "a\n") + two = write(self.root / "two", "b\n") + self.assertEqual(answer("secret_set_verdict", str(one), str(two)), "readable") + + def test_a_missing_file_outranks_a_refusal_and_names_every_absence(self) -> None: + refused = write(self.root / "refused", "a\n", 0o644) + self.assertEqual( + answer( + "secret_set_verdict", + str(refused), + str(self.root / "first"), + str(self.root / "second"), + ), + "missing first second", + ) + + def test_the_first_refusal_is_the_one_named(self) -> None: + first = write(self.root / "first", "a\n", 0o644) + second = write(self.root / "second", "b\r\n") + self.assertEqual( + answer("secret_set_verdict", str(first), str(second)), + f"refused {first} is mode 644, and must give group and other no access", + ) + + +class RequireSecret(Tree): + def test_a_readable_secret_succeeds_silently(self) -> None: + key = write(self.root / "key", "value\n") + result = run('require_secret "$1"', str(key)) + self.assertEqual((result.returncode, result.stdout, result.stderr), (0, "", "")) + + def test_an_absent_secret_fails_and_says_so(self) -> None: + result = run('require_secret "$1"', str(self.root / "key")) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stderr, f"ERROR: {self.root}/key is not there\n") + + def test_a_refused_secret_fails_with_the_reason(self) -> None: + key = write(self.root / "key", "value\n", 0o604) + result = run('require_secret "$1"', str(key)) + self.assertEqual(result.returncode, 1) + self.assertEqual( + result.stderr, f"ERROR: {key} is mode 604, and must give group and other no access\n" + ) + + def test_a_bare_call_stops_a_script_under_errexit(self) -> None: + result = run('require_secret "$1"; echo reached', str(self.root / "key")) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("reached", result.stdout) + + +class ReadEnvValue(Tree): + def test_reads_the_named_value(self) -> None: + env = write(self.root / "app.env", "APP_TOKEN=one\nAPP_HOST=two\n") + self.assertEqual(answer("read_env_value", str(env), "APP_HOST"), "two") + + def test_an_unset_name_reads_empty(self) -> None: + env = write(self.root / "app.env", "APP_TOKEN=one\n") + self.assertEqual(answer("read_env_value", str(env), "APP_HOST"), "") + + def test_the_caller_environment_does_not_stand_in_for_the_file(self) -> None: + env = write(self.root / "app.env", "APP_TOKEN=one\n") + result = run( + 'read_env_value "$1" APP_HOST', str(env), env={**os.environ, "APP_HOST": "stale"} + ) + self.assertEqual((result.returncode, result.stdout), (0, "")) + + def test_a_failing_line_anywhere_fails_the_read(self) -> None: + env = write(self.root / "app.env", "false\nAPP_HOST=two\n") + result = run('read_env_value "$1" APP_HOST', str(env)) + self.assertNotEqual(result.returncode, 0) + + def test_a_failing_line_fails_inside_an_if_condition(self) -> None: + env = write(self.root / "app.env", "false\nAPP_HOST=two\n") + result = run( + 'if read_env_value "$1" APP_HOST; then echo ok; else echo failed; fi', str(env) + ) + self.assertEqual(result.stdout, "failed\n") + + def test_what_the_file_prints_stays_out_of_the_value(self) -> None: + env = write(self.root / "app.env", "echo noise\nAPP_HOST=two\n") + self.assertEqual(answer("read_env_value", str(env), "APP_HOST"), "two") + + def test_the_file_cannot_redirect_the_read_through_the_positional_parameters(self) -> None: + env = write(self.root / "app.env", "APP_HOST=two\nAPP_OTHER=wrong\nset -- x APP_OTHER\n") + self.assertEqual(answer("read_env_value", str(env), "APP_HOST"), "two") + + def test_nothing_the_file_sets_reaches_the_caller(self) -> None: + env = write(self.root / "app.env", "APP_TOKEN=one\nAPP_HOST=two\n") + result = run( + 'value=$(read_env_value "$1" APP_HOST); printf "%s|%s" "$value" "${APP_TOKEN-unset}"', + str(env), + ) + self.assertEqual(result.stdout, "two|unset") + + def test_bash_env_does_not_run_ahead_of_the_file(self) -> None: + # A function shadows the builtin, so a hook that ran in the reading bash would answer for the file. + # The outer bash runs it too, and loads the library with `.` rather than `source` for that reason. + hook = write(self.root / "hook.sh", "source() { APP_HOST=injected; }\n") + env = write(self.root / "app.env", "APP_TOKEN=one\n") + result = run( + 'read_env_value "$1" APP_HOST', str(env), env={**os.environ, "BASH_ENV": str(hook)} + ) + self.assertEqual((result.returncode, result.stdout), (0, "")) + + +if __name__ == "__main__": + unittest.main() From ee58dec381b15f3b975af997fb141b3b87863220 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 13:06:19 -0700 Subject: [PATCH 02/17] Answer the Local Review of the Secret Readers Change The carried-trees check now reuses carry.py inventory, compare, and included, so include and prune read as a carry reads them, and it knows a hub by its tracked spec/files.json. The Hub-Hosted Tooling paragraph no longer claims the audit refreshes a copy or that hub_fetch_run.py is carried by its reasoning. Repo-Scoped Secrets states the reader rule the host-secrets check judges. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 85 ++++++++++++-------------- GOVERNANCE.md | 3 +- scripts/README.md | 2 +- spec/project-types.json | 2 +- spec/type-model.md | 2 +- tests/test_repo_gate.py | 54 ++++++++++------ tests/test_secrets_sh.py | 4 +- 7 files changed, 83 insertions(+), 69 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 9f5c0f48a..2427227e2 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -36,6 +36,7 @@ from __future__ import annotations import argparse +import importlib.util import json import os import re @@ -43,6 +44,7 @@ import subprocess import sys from pathlib import Path +from types import ModuleType # GOVERNANCE.md documents exactly one floating-ref exception. SHA_EXCEPTIONS = {"dotnet/nbgv"} @@ -785,19 +787,16 @@ def gate_hub_root() -> Path | None: return root if (root / "spec" / "files.json").is_file() else None -def workflow_command_data(text: str) -> str: - """`text` escaped as a workflow command's message, per the runner's own percent encoding.""" - return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") +def hub_carry(hub: Path) -> ModuleType: + """The hub's own `scripts/carry.py`, whose tree inventory and comparison this check reuses. - -def hub_tree_files(source: Path) -> dict[str, bytes]: - """Every file under `source`, keyed by its path relative to it, never following a link.""" - found: dict[str, bytes] = {} - for current, _, filenames in os.walk(source, followlinks=False): - for name in filenames: - path = Path(current) / name - found[path.relative_to(source).as_posix()] = path.read_bytes() - return found + It imports only the standard library until a section is read, so plain `python3` loads it. + """ + spec = importlib.util.spec_from_file_location("carry", hub / "scripts" / "carry.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module def check_carried_trees(root: Path, files: list[str]) -> list[str]: @@ -806,58 +805,52 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: `spec/files.json` declares the trees a repository carries byte-for-byte, and a carried copy goes stale silently at the next hub fix. Comparing against the hub tree this gate runs from makes the answer reproducible at the action's pin, so a Dependabot bump of that pin is what - surfaces a refresh the audit and resync then apply. + surfaces a refresh the resync then applies. A difference is a note rather than a finding, so it never fails the gate. Failing would break every downstream pull request on each hub fix, and a byte difference cannot tell a stale copy - from an edited one, which the audit tells apart and disposes of. Which trees a repository owes - is also the audit's, read from its declared types, so a tree absent here is not compared. - The target side is read from the tracked files alone, so an untracked file is not a difference. + from an edited one, which the audit tells apart. Which trees a repository owes is the audit's + too, read from its declared types, so a tree absent here is not compared. The copy is read + from its tracked files alone, so an untracked file is not a difference. """ hub = gate_hub_root() if hub is None: NOTES.append( - "no hub tree is beside this gate, as with a single-file fetch, so no carried tree was compared." + "no hub tree is beside this gate, as with a single-file fetch, " + "so no carried tree was compared." ) return [] - if hub == root.resolve(): - NOTES.append( - "the scanned repository is the hub, whose trees are the sources rather than copies." - ) + # A hub tracks its manifest, which no carrying repository does, so a hub worktree on another branch is not read as a stale copy. + if "spec/files.json" in files: + NOTES.append("the scanned repository is a hub, whose trees are sources rather than copies.") return [] - try: - trees = json.loads((hub / "spec" / "files.json").read_text(encoding="utf-8"))["trees"] - except (OSError, ValueError, KeyError) as error: - return [f"spec/files.json in the hub tree could not be read: {error}"] + carry = hub_carry(hub) carried = 0 - for tree in trees: - target = tree["target"].rstrip("/") + for tree in carry.load_json(hub / "spec" / "files.json")["trees"]: + target, patterns = tree["target"], tree["include"] prefix = target + "/" - tracked_here = [path for path in files if path.startswith(prefix)] + tracked_here = [path[len(prefix) :] for path in files if path.startswith(prefix)] if not tracked_here: continue carried += 1 - if tree["include"] != ["**/*"]: - NOTES.append( - f"{target} declares include patterns this check does not read, so it was not compared." - ) - continue - source = hub_tree_files(hub / tree["source"]) copy: dict[str, bytes | None] = {} - for path in tracked_here: - on_disk = root / path - copy[path[len(prefix) :]] = ( - None if on_disk.is_symlink() or not on_disk.is_file() else on_disk.read_bytes() - ) - missing = sorted(set(source) - set(copy)) - extra = sorted(set(copy) - set(source)) - changed = sorted(path for path in set(source) & set(copy) if copy[path] != source[path]) - if missing or extra or changed: - # Shown on the pull request rather than only in the log, from the manifest's own paths, so nothing a scanned repository names reaches the command. + for relative in tracked_here: + if carry.included(relative, patterns): + path = root / target / relative + copy[relative] = ( + None if path.is_symlink() or not path.is_file() else path.read_bytes() + ) + result = carry.compare( + carry.inventory(hub / tree["source"], patterns), + carry.Inventory(copy, frozenset(), ""), + ) + changed, missing = result["modified"], result["missing"] + # A tree that does not prune leaves the repository's own files under it alone, as a carry does. + extra = result["extra"] if tree["prune"] else [] + if changed or missing or extra: if os.environ.get("GITHUB_ACTIONS") == "true": print( - "::warning title=carried-trees::" - + workflow_command_data(f"{target} differs from the hub's {tree['source']}") + f"::warning title=carried-trees::{target} differs from the hub's {tree['source']}" ) NOTES.append( f"{target} differs from the hub's {tree['source']}: {len(changed)} changed, " diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 922bc15f4..b933f621d 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -123,6 +123,7 @@ A repo whose own stacks or scripts read local runtime credentials from disk, mos A fresh checkout then documents its own required shape without ever exposing a real value. This negation keeps a real secret file out of a **new** commit. It does not remove one already tracked: `.gitignore` has no effect on a path git already follows. A real secret file found tracked is removed from the index (`git rm --cached `) and its credential is rotated, not just added to `.gitignore` going forward. - **`.secrets/README.md` is a catalog**, one row per file naming what it holds and what consumes it, plus a short note on how the directory relates to `~/.secrets/` where the repo also touches that. +- **A repository whose shell scripts read a secret or env file declares the `host-secrets` type, and reads each one through the carried readers.** A single inline read counts, since it is the read a bare existence check and a sourced file get wrong. A script sources the hub's `lib/fleet/sh/secrets.sh` and checks the file with `require_secret`, `secret_verdict`, or `secret_set_verdict` before reading it. It reads one value from an env file with `read_env_value` rather than sourcing the file into its own shell, per "Hub-Hosted Tooling". ## Repository Onboarding and Conformance @@ -139,7 +140,7 @@ The fleet's tooling lives in the hub once and a repository runs it from there ra **What a repository carries and what it reaches is decided by what the content is.** It carries the content it is audited against and the configuration that describes it, meaning its rule text and the files the manifest declares. It reaches machinery whose content is identical in every repository, meaning the prose and repository gates, the review digest, and the configuration script, because a file holding no per-repo content is a copy whose only future is to go stale. A tool named in a carried rule is therefore named as the hub's, since the alternative is a pointer to a path the reader does not have, and a pointer that resolves nowhere teaches the reader that a pointer in carried text is decorative. -**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root, so fetching it from the hub at run time lets a GitHub outage or a bad `main` commit stop a backup, and it runs code the repository never reviewed against its own secrets. Offline operation and review in the repository that runs it outweigh the staleness of a copy there, which is not true of a gate, where "the tool did not run" is an acceptable answer and CI is the backstop. The hub hosts the library once, and a repository whose declared types select it carries it byte-for-byte as a tree the hub's `spec/files.json` declares. The repository extends it in a file of its own that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds and reports a difference as a warning that never fails it, since a failure would break every downstream pull request on each hub fix, and the audit and the resync apply the refresh. A local hook's `hub_fetch_run.py` is carried by the same reasoning, since it is the loader that reaches the hub and so cannot itself be reached. +**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root. Fetched from the hub at run time, it would let a GitHub outage or a bad `main` commit stop a backup. It would also run code the repository never reviewed against its own secrets. So offline operation and review in the repository that runs it outweigh the staleness of a copy. A gate differs, since "the tool did not run" is an acceptable answer there and CI is the backstop. The hub hosts the library once. A repository whose declared types select it carries it byte-for-byte, as a tree the hub's `spec/files.json` declares. It extends the library in a file of its own that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds. It reports a difference as a warning that never fails the gate. A failure would break every downstream pull request on each hub fix. The audit reports a stale copy, and the resync refreshes it. **Reaching it is a checkout of the hub rather than a copy of one file.** A tool reads the payloads, tables, and sibling modules beside it, so a single file lifted out of the tree runs against whatever the caller happens to have, which is the copy problem again in a shorter loop. Read `main`, the promoted and gated state, and fetch immediately before running, because a clone is whatever it last fetched rather than the branch it names, and a stale clone answers confidently instead of failing. Name the tool by its path in that checkout and name the target explicitly, since a tool that defaults to the current directory or the current repository resolves somewhere either way, and a result computed against the wrong repository is well-formed. Which directory the command runs in is the tool's own contract rather than a rule here, so a gate reading a working tree runs in the repository under test while a tool taking its target as an argument runs anywhere and is given one. What the rule forbids is letting a default decide which repository the answer is about. diff --git a/scripts/README.md b/scripts/README.md index 8f8111012..f5752b8b7 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares that the scanned repository tracks at its target is compared byte-for-byte with the hub tree beside the gate, the pinned action's tree in CI and the checkout in a local run. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions, and never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit gives each its disposition and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a hub `main` newer than the pin also warns until Dependabot advances it. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it and says so in a note, and the hub scanning itself compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate, wherever the scanned repository tracks files at its target. The hub tree is the pinned action's in CI and the checkout's in a local run, and the comparison reuses [`carry.py`](#carrypy)'s inventory, so `include` and `prune` read the same way. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/spec/project-types.json b/spec/project-types.json index f468208d7..c7f4c267c 100644 --- a/spec/project-types.json +++ b/spec/project-types.json @@ -134,7 +134,7 @@ "host-secrets": { "detect": ["lib/fleet/sh/secrets.sh", "a shell script the repo runs that sources or reads a secret or env file from disk"], "checks": [ - { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a secret or env file sources the carried lib/fleet/sh/secrets.sh and checks the file with require_secret, secret_verdict, or secret_set_verdict before reading it, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file that sources the library, never in an edit to the carried copy, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#hub-hosted-tooling" } + { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a secret or env file sources the carried lib/fleet/sh/secrets.sh and checks the file with require_secret, secret_verdict, or secret_set_verdict before reading it, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file that sources the library, never in an edit to the carried copy, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#repo-scoped-secrets" } ] } }, diff --git a/spec/type-model.md b/spec/type-model.md index bf2663ba7..0405731ac 100644 --- a/spec/type-model.md +++ b/spec/type-model.md @@ -58,7 +58,7 @@ A generator type declares no `profiles`. Build versus lint-only is a depth of *l ## Runtime Libraries -A **runtime library** type declares that the repo's own runtime executes a library the hub hosts, so the type's token is what selects the library's tree in [`spec/files.json`][files] and the repo carries it byte-for-byte. `host-secrets` is the one such type, for a repo whose scripts read secret and env files on a host it owns, and it carries `lib/sh` from the hub to `lib/fleet/sh`. The type is declared rather than inferred like any other, since a repo that reads one secret inline and one that runs a fleet of host services look alike to a file scan. A runtime library type declares no `profiles`, for the reason a generator type declares none. +A **runtime library** type declares that the repo's own runtime executes a library the hub hosts. The type's token selects the library's tree in [`spec/files.json`][files], and the repo carries it byte-for-byte. `host-secrets` is the one such type, for a repo whose shell scripts read a secret or env file, and it carries `lib/sh` from the hub to `lib/fleet/sh`. Its `detect` entry surfaces such a script as a discovery advisory, and the declaration is what makes the type's check apply, as for any type. A runtime library type declares no `profiles`, for the reason a generator type declares none. ## Changing the type set carries review weight diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 3ea2e4247..9b4f8a608 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1345,10 +1345,21 @@ def setUp(self) -> None: super().setUp() self.hub = self.tmp / "hub" self.repo = self.tmp / "repo" - self.manifest([{"source": "lib/sh", "target": "lib/fleet/sh", "include": ["**/*"]}]) + self.manifest([self.tree()]) self.write(self.hub / "lib/sh/secrets.sh", "hub\n") + (self.hub / "scripts").mkdir() + shutil.copy(REPO / "scripts" / "carry.py", self.hub / "scripts" / "carry.py") self.enterContext(mock.patch.object(repo_gate, "gate_hub_root", return_value=self.hub)) + @staticmethod + def tree(include: list[str] | None = None, prune: bool = True) -> dict[str, object]: + return { + "source": "lib/sh", + "target": "lib/fleet/sh", + "include": include or ["**/*"], + "prune": prune, + } + def manifest(self, trees: list[dict[str, object]]) -> None: self.write(self.hub / "spec/files.json", json.dumps({"trees": trees})) @@ -1378,7 +1389,8 @@ def test_a_changed_byte_is_a_note_and_never_a_finding(self) -> None: hits, _ = self.run_check(files) self.assertEqual(hits, []) self.assertIn( - "lib/fleet/sh differs from the hub's lib/sh: 1 changed, 0 missing, 0 extra, first secrets.sh.", + "lib/fleet/sh differs from the hub's lib/sh: 1 changed, 0 missing, 0 extra, " + "first secrets.sh.", repo_gate.NOTES[0], ) @@ -1387,6 +1399,15 @@ def test_a_missing_and_an_extra_file_are_counted(self) -> None: self.run_check(files) self.assertIn("0 changed, 1 missing, 1 extra, first secrets.sh.", repo_gate.NOTES[0]) + def test_a_tree_that_does_not_prune_leaves_a_repository_file_alone(self) -> None: + self.manifest([self.tree(prune=False)]) + files = [ + self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n"), + self.write(self.repo / "lib/fleet/sh/local.sh", "mine\n"), + ] + self.run_check(files) + self.assertEqual(len(repo_gate.NOTES), 1) + def test_an_untracked_file_is_not_a_difference(self) -> None: files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] self.write(self.repo / "lib/fleet/sh/scratch.sh", "untracked\n") @@ -1405,19 +1426,15 @@ def test_a_tree_the_repository_does_not_carry_is_not_compared(self) -> None: repo_gate.NOTES, ["compared 0 carried tree(s) against the hub tree beside this gate."] ) - def test_a_narrowed_include_is_named_rather_than_compared(self) -> None: - self.manifest([{"source": "lib/sh", "target": "lib/fleet/sh", "include": ["*.sh"]}]) - self.run_check([self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")]) - self.assertEqual( - repo_gate.NOTES[0], - "lib/fleet/sh declares include patterns this check does not read, so it was not compared.", - ) - - def test_an_unreadable_manifest_is_a_finding(self) -> None: - self.write(self.hub / "spec/files.json", "{") - hits, _ = self.run_check([]) - self.assertEqual(len(hits), 1) - self.assertTrue(hits[0].startswith("spec/files.json in the hub tree could not be read")) + def test_a_narrowed_include_compares_only_what_it_names(self) -> None: + self.manifest([self.tree(include=["*.sh"])]) + self.write(self.hub / "lib/sh/README.txt", "hub only\n") + files = [ + self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n"), + self.write(self.repo / "lib/fleet/sh/notes.txt", "repository only\n"), + ] + self.run_check(files) + self.assertEqual(len(repo_gate.NOTES), 1) def test_actions_shows_the_difference_on_the_pull_request(self) -> None: files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")] @@ -1436,11 +1453,12 @@ def test_outside_actions_nothing_is_printed(self) -> None: files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")] self.assertEqual(self.run_check(files)[1], "") - def test_the_hub_scanning_itself_compares_nothing(self) -> None: - self.assertEqual(repo_gate.check_carried_trees(self.hub, []), []) + def test_a_hub_on_another_branch_is_not_read_as_a_stale_copy(self) -> None: + files = ["spec/files.json", self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")] + self.assertEqual(self.run_check(files), ([], "")) self.assertEqual( repo_gate.NOTES, - ["the scanned repository is the hub, whose trees are the sources rather than copies."], + ["the scanned repository is a hub, whose trees are sources rather than copies."], ) diff --git a/tests/test_secrets_sh.py b/tests/test_secrets_sh.py index fd2bab838..8ca3feefa 100755 --- a/tests/test_secrets_sh.py +++ b/tests/test_secrets_sh.py @@ -12,6 +12,7 @@ import os import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -19,7 +20,7 @@ ROOT = Path(__file__).resolve().parent.parent LIBRARY = ROOT / "lib" / "sh" / "secrets.sh" TIMEOUT = 30 -ROOT_ACCOUNT = os.geteuid() == 0 +ROOT_ACCOUNT = hasattr(os, "geteuid") and os.geteuid() == 0 def run( @@ -50,6 +51,7 @@ def write(path: Path, content: str, mode: int = 0o600) -> Path: return path +@unittest.skipUnless(sys.platform == "linux", "the library runs under bash on a Linux host") class Tree(unittest.TestCase): def setUp(self) -> None: self.temp = tempfile.TemporaryDirectory() From 927219b79b5b494c229947f0af1edb9e58132fd9 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 13:21:22 -0700 Subject: [PATCH 03/17] Answer the Second Local Review of the Secret Readers Change The carried-trees check reads a pruned tree as owning every file under its target, as a carry and the audit do, and a tree it cannot compare becomes a note rather than a traceback. The reader rule names the credential files it binds, under .secrets/ or ~/.secrets/, and which verdict takes the shared mode. The env read isolation test now calls the reader in the outer shell. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 28 +++++++++++++++++--------- .github/workflows/validate-task.yml | 2 +- AUDIT.md | 2 +- GOVERNANCE.md | 4 ++-- scripts/README.md | 4 ++-- spec/fidelity-model.md | 2 +- spec/project-types.json | 4 ++-- spec/type-model.md | 2 +- tests/test_cli_usage.py | 4 ++-- tests/test_repo_gate.py | 17 ++++++++++++++-- tests/test_secrets_sh.py | 4 ++-- 11 files changed, 47 insertions(+), 26 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 2427227e2..dcf61c3a4 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -833,17 +833,25 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: if not tracked_here: continue carried += 1 - copy: dict[str, bytes | None] = {} - for relative in tracked_here: - if carry.included(relative, patterns): - path = root / target / relative - copy[relative] = ( - None if path.is_symlink() or not path.is_file() else path.read_bytes() - ) - result = carry.compare( - carry.inventory(hub / tree["source"], patterns), - carry.Inventory(copy, frozenset(), ""), + # A pruned tree owns everything under its target, as a carry and the audit read it, so `include` narrows only one that does not prune. + owned = ( + tracked_here + if tree["prune"] + else [p for p in tracked_here if carry.included(p, patterns)] ) + try: + copy = { + relative: None + if (root / target / relative).is_symlink() + or not (root / target / relative).is_file() + else (root / target / relative).read_bytes() + for relative in owned + } + source = carry.inventory(hub / tree["source"], patterns) + except (carry.CarryError, OSError) as error: + NOTES.append(f"{target} could not be compared: {error}") + continue + result = carry.compare(source, carry.Inventory(copy, frozenset(), "")) changed, missing = result["modified"], result["missing"] # A tree that does not prune leaves the repository's own files under it alone, as a carry does. extra = result["extra"] if tree["prune"] else [] diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index 6a53138cf..79fed87a9 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -23,7 +23,7 @@ on: default: '' # Threaded to the repo-gate composite action's own exclude-globs input, unvalidated. repo-gate-exclude-globs: - description: Git pathspec patterns to exclude, one per line (for example 'themes/PaperMod/**'), narrowing the sha-pin, eol-coverage, and composite-actions scans via repo_gate.py's --exclude. + description: Git pathspec patterns to exclude, one per line (for example 'themes/PaperMod/**'), narrowing the sha-pin, eol-coverage, composite-actions, and carried-trees scans via repo_gate.py's --exclude. required: false type: string default: '' diff --git a/AUDIT.md b/AUDIT.md index 6679468a9..98ff8b33d 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -58,7 +58,7 @@ Otherwise read its `types[]`. If the entry is `classificationPending` (a backlog - `Dockerfile` + a docker build/push workflow -> `docker`, an `upstream-version.json` tracker -> `upstream-wrapper`. - `custom_components/*/manifest.json` + `hacs.json` -> `homeassistant`, a codegen workflow -> `codegen`, no `build-*` task -> `source-only`, governance-only -> `docs`. - `hugo.yaml` / `hugo.toml` / `config/_default/hugo.yaml` -> `hugo`. A repo may carry it alongside `source-only`, since a site deploy leaf is not a `build-*` task and both declarations stay true. -- a carried `lib/fleet/sh/secrets.sh`, or a shell script the repo runs that sources or reads a secret or env file from disk -> `host-secrets`, which selects that carried tree. +- a carried `lib/fleet/sh/secrets.sh`, or a shell script the repo runs that sources or reads a credential file under `.secrets/` or `~/.secrets/` -> `host-secrets`, which selects that carried tree. ## 3. Applicability Gate diff --git a/GOVERNANCE.md b/GOVERNANCE.md index b933f621d..a4f96036f 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -123,7 +123,7 @@ A repo whose own stacks or scripts read local runtime credentials from disk, mos A fresh checkout then documents its own required shape without ever exposing a real value. This negation keeps a real secret file out of a **new** commit. It does not remove one already tracked: `.gitignore` has no effect on a path git already follows. A real secret file found tracked is removed from the index (`git rm --cached `) and its credential is rotated, not just added to `.gitignore` going forward. - **`.secrets/README.md` is a catalog**, one row per file naming what it holds and what consumes it, plus a short note on how the directory relates to `~/.secrets/` where the repo also touches that. -- **A repository whose shell scripts read a secret or env file declares the `host-secrets` type, and reads each one through the carried readers.** A single inline read counts, since it is the read a bare existence check and a sourced file get wrong. A script sources the hub's `lib/fleet/sh/secrets.sh` and checks the file with `require_secret`, `secret_verdict`, or `secret_set_verdict` before reading it. It reads one value from an env file with `read_env_value` rather than sourcing the file into its own shell, per "Hub-Hosted Tooling". +- **A repository whose shell scripts read a credential from disk declares the `host-secrets` type, and reads each one through the carried readers.** A credential is a secret file or an env file holding one, in `.secrets/` or in the host-scoped `~/.secrets/`, and this bullet reaches both. A single inline read counts, since it is the read a bare existence check and a sourced file get wrong. A script sources its carried `lib/fleet/sh/secrets.sh` and checks the file before reading it. A private file takes `require_secret` or `secret_set_verdict`, and one a container reads as a Docker secret takes `secret_verdict` with `shared`. It reads one value from an env file with `read_env_value` rather than sourcing the file into its own shell, per "Hub-Hosted Tooling". ## Repository Onboarding and Conformance @@ -140,7 +140,7 @@ The fleet's tooling lives in the hub once and a repository runs it from there ra **What a repository carries and what it reaches is decided by what the content is.** It carries the content it is audited against and the configuration that describes it, meaning its rule text and the files the manifest declares. It reaches machinery whose content is identical in every repository, meaning the prose and repository gates, the review digest, and the configuration script, because a file holding no per-repo content is a copy whose only future is to go stale. A tool named in a carried rule is therefore named as the hub's, since the alternative is a pointer to a path the reader does not have, and a pointer that resolves nowhere teaches the reader that a pointer in carried text is decorative. -**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root. Fetched from the hub at run time, it would let a GitHub outage or a bad `main` commit stop a backup. It would also run code the repository never reviewed against its own secrets. So offline operation and review in the repository that runs it outweigh the staleness of a copy. A gate differs, since "the tool did not run" is an acceptable answer there and CI is the backstop. The hub hosts the library once. A repository whose declared types select it carries it byte-for-byte, as a tree the hub's `spec/files.json` declares. It extends the library in a file of its own that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds. It reports a difference as a warning that never fails the gate. A failure would break every downstream pull request on each hub fix. The audit reports a stale copy, and the resync refreshes it. +**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root. Fetched from the hub at run time, it would let a GitHub outage or a bad `main` commit stop a scheduled host job. It would also run code the repository never reviewed against its own secrets. So offline operation and review in the repository that runs it outweigh the staleness of a copy. A gate differs, since "the tool did not run" is an acceptable answer there and CI is the backstop. The hub hosts the library once. A repository whose declared types select it carries it byte-for-byte, as a tree the hub's `spec/files.json` declares. It extends the library in a file of its own that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds. It reports a difference as a warning that never fails the gate. A failure would break every downstream pull request on each hub fix. The audit reports a stale copy, and the resync refreshes it. **Reaching it is a checkout of the hub rather than a copy of one file.** A tool reads the payloads, tables, and sibling modules beside it, so a single file lifted out of the tree runs against whatever the caller happens to have, which is the copy problem again in a shorter loop. Read `main`, the promoted and gated state, and fetch immediately before running, because a clone is whatever it last fetched rather than the branch it names, and a stale clone answers confidently instead of failing. Name the tool by its path in that checkout and name the target explicitly, since a tool that defaults to the current directory or the current repository resolves somewhere either way, and a result computed against the wrong repository is well-formed. Which directory the command runs in is the tool's own contract rather than a rule here, so a gate reading a working tree runs in the repository under test while a tool taking its target as an argument runs anywhere and is given one. What the rule forbids is letting a default decide which repository the answer is about. diff --git a/scripts/README.md b/scripts/README.md index f5752b8b7..6611c9b16 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate, wherever the scanned repository tracks files at its target. The hub tree is the pinned action's in CI and the checkout's in a local run, and the comparison reuses [`carry.py`](#carrypy)'s inventory, so `include` and `prune` read the same way. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate, wherever the scanned repository tracks files at its target. The hub tree is the pinned action's in CI and the checkout's in a local run, and the comparison reuses [`carry.py`](#carrypy)'s inventory, so `include` and `prune` read as a carry reads them. A tree that cannot be compared, such as one whose hub source holds a link, says so in a note. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py @@ -154,7 +154,7 @@ The resolution pass is **scoped to the scanned repository's own owner**, which i A pin is a finding only where GitHub **answered** that the commit is absent, meaning a 404 or a 422. An offline host, a missing `gh`, a rate limit and a narrow token all report as unread and leave the pin on its shape, so the gate stays usable with no network instead of failing a correct tree. A 404 is confirmed against the repository itself before it becomes a finding, since an absent commit and a repository the credentials cannot see are the same answer from here, and a repository-scoped CI token is narrower than the fleet. That second read runs only on the failing path, and each distinct pin is read once however often it repeats. -A `note:` line is how a check says it did less than its name. It prints under the check's own result, outside the issue count, and never changes the exit code, because nothing is wrong with the tree when the network is what is missing. It carries one fixed shape every run so a zero is as visible as a count, and it separates a pin under a known other owner from one whose owner could not be compared at all, since a checkout with no readable `origin` skips every pin including this owner's own and reporting those as somebody else's is the same false clean the note was added to prevent. +A `note:` line is how a check says it did less than its name, or, for `carried-trees`, how it reports a difference that is deliberately not a finding. It prints under the check's own result, outside the issue count, and never changes the exit code, because nothing is wrong with the tree when the network is what is missing. It carries one fixed shape every run so a zero is as visible as a count, and it separates a pin under a known other owner from one whose owner could not be compared at all, since a checkout with no readable `origin` skips every pin including this owner's own and reporting those as somebody else's is the same false clean the note was added to prevent. A stale-backticked-path check was built and **rejected**: a template repo legitimately references paths that live in downstream repos, so it produced 34 false positives on a clean tree with no way to separate those from real drift. Doc-to-doc drift is a review lens, not a regex. diff --git a/spec/fidelity-model.md b/spec/fidelity-model.md index 920331b31..b9629483e 100644 --- a/spec/fidelity-model.md +++ b/spec/fidelity-model.md @@ -52,7 +52,7 @@ So a local rule edit to `CODESTYLE.md` or `WORKFLOW.md`, both `intent` and whole A verbatim check compares content by hash after **line-ending, action-pin, and job-needs normalization**. EOL variance is governed by the line-ending rules, a `uses: @` pin (with its trailing `# vN` comment) is Dependabot-owned and bumped per repo, and a job's `needs:` list is pruned per repo to its vendored targets (an unvendored name fails the workflow to load), so all three are governed drift rather than a fidelity deviation. This keeps a verbatim workflow job region (the `github-release` job) from flagging on a routine action bump or a legitimate needs-prune while still catching a real structural fork. It does **not** mask placeholders: a verbatim unit carries none. The files that declare a `placeholders` list (for example `.github/copilot-instructions.md` with ``, ``, ``) are fidelity `intent`, judged by hand and never hashed. Masking could not serve a hash anyway. A downstream copy holds the substituted value (`ptr727`), not the token (``), so masking the token in the canonical alone would guarantee a mismatch. A verbatim unit that ever needed a per-repo substitution would require template-matching (the canonical as a pattern, the copy as an instance), not this content hash. None does today. -`verbatim-tree` applies no normalization. It compares file bytes because a fully owned generated tree has no per-repository substitutions or Dependabot-owned regions. +`verbatim-tree` applies no normalization. It compares file bytes because a fully owned tree has no per-repository substitutions or Dependabot-owned regions. ## Stale Versus Modified diff --git a/spec/project-types.json b/spec/project-types.json index c7f4c267c..5eef339ef 100644 --- a/spec/project-types.json +++ b/spec/project-types.json @@ -132,9 +132,9 @@ ] }, "host-secrets": { - "detect": ["lib/fleet/sh/secrets.sh", "a shell script the repo runs that sources or reads a secret or env file from disk"], + "detect": ["lib/fleet/sh/secrets.sh", "a shell script the repo runs that sources or reads a credential file under .secrets/ or ~/.secrets/"], "checks": [ - { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a secret or env file sources the carried lib/fleet/sh/secrets.sh and checks the file with require_secret, secret_verdict, or secret_set_verdict before reading it, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file that sources the library, never in an edit to the carried copy, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#repo-scoped-secrets" } + { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a credential from disk, a secret file or an env file holding one under .secrets/ or ~/.secrets/, sources the carried lib/fleet/sh/secrets.sh and checks the file with require_secret, secret_verdict, or secret_set_verdict before reading it, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file that sources the library, never in an edit to the carried copy, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#repo-scoped-secrets" } ] } }, diff --git a/spec/type-model.md b/spec/type-model.md index 0405731ac..50feea612 100644 --- a/spec/type-model.md +++ b/spec/type-model.md @@ -58,7 +58,7 @@ A generator type declares no `profiles`. Build versus lint-only is a depth of *l ## Runtime Libraries -A **runtime library** type declares that the repo's own runtime executes a library the hub hosts. The type's token selects the library's tree in [`spec/files.json`][files], and the repo carries it byte-for-byte. `host-secrets` is the one such type, for a repo whose shell scripts read a secret or env file, and it carries `lib/sh` from the hub to `lib/fleet/sh`. Its `detect` entry surfaces such a script as a discovery advisory, and the declaration is what makes the type's check apply, as for any type. A runtime library type declares no `profiles`, for the reason a generator type declares none. +A **runtime library** type declares that the repo's own runtime executes a library the hub hosts. The type's token selects the library's tree in [`spec/files.json`][files], and the repo carries it byte-for-byte. `host-secrets` is the one such type, for a repo whose shell scripts read a credential from disk, and it carries `lib/sh` from the hub to `lib/fleet/sh`. Its `detect` entry surfaces such a script as a discovery advisory, and the declaration is what makes the type's check apply, as for any type. A runtime library type declares no `profiles`, for the reason a generator type declares none. ## Changing the type set carries review weight diff --git a/tests/test_cli_usage.py b/tests/test_cli_usage.py index 216491d28..78301f49f 100755 --- a/tests/test_cli_usage.py +++ b/tests/test_cli_usage.py @@ -24,8 +24,8 @@ tree check covers ignored paths and a directory created or emptied anywhere in the tree. The set is discovered from the tree rather than listed, so a new script is held to the rule the -moment it is tracked. An extensionless file opening on a shebang counts. A hook its own host runs -is not a command-line entry point and is listed in NOT_COMMAND_LINE. +moment it is tracked. An extensionless file opening on a shebang counts. A hook its own host runs, +and a library a script sources, is not a command-line entry point and is listed in NOT_COMMAND_LINE. A process the run leaves behind is found by a token in the environment it inherited. That token is read from /proc, so the check runs only on a host that has /proc. On Windows only the PowerShell diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 9b4f8a608..4743dc26a 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1426,16 +1426,29 @@ def test_a_tree_the_repository_does_not_carry_is_not_compared(self) -> None: repo_gate.NOTES, ["compared 0 carried tree(s) against the hub tree beside this gate."] ) - def test_a_narrowed_include_compares_only_what_it_names(self) -> None: - self.manifest([self.tree(include=["*.sh"])]) + def narrowed(self, prune: bool) -> None: + self.manifest([self.tree(include=["*.sh"], prune=prune)]) self.write(self.hub / "lib/sh/README.txt", "hub only\n") files = [ self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n"), self.write(self.repo / "lib/fleet/sh/notes.txt", "repository only\n"), ] self.run_check(files) + + def test_a_narrowed_include_narrows_the_hub_side_only_of_a_pruned_tree(self) -> None: + self.narrowed(prune=True) + self.assertIn("0 changed, 0 missing, 1 extra, first notes.txt.", repo_gate.NOTES[0]) + + def test_a_narrowed_include_narrows_both_sides_of_a_tree_that_does_not_prune(self) -> None: + self.narrowed(prune=False) self.assertEqual(len(repo_gate.NOTES), 1) + def test_a_tree_that_cannot_be_compared_is_a_note_and_never_a_finding(self) -> None: + (self.hub / "lib/sh/link").symlink_to(self.hub / "lib/sh/secrets.sh") + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] + self.assertEqual(self.run_check(files), ([], "")) + self.assertTrue(repo_gate.NOTES[0].startswith("lib/fleet/sh could not be compared: ")) + def test_actions_shows_the_difference_on_the_pull_request(self) -> None: files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")] _, printed = self.run_check(files, actions=True) diff --git a/tests/test_secrets_sh.py b/tests/test_secrets_sh.py index 8ca3feefa..3b1eb2fc1 100755 --- a/tests/test_secrets_sh.py +++ b/tests/test_secrets_sh.py @@ -302,10 +302,10 @@ def test_the_file_cannot_redirect_the_read_through_the_positional_parameters(sel def test_nothing_the_file_sets_reaches_the_caller(self) -> None: env = write(self.root / "app.env", "APP_TOKEN=one\nAPP_HOST=two\n") result = run( - 'value=$(read_env_value "$1" APP_HOST); printf "%s|%s" "$value" "${APP_TOKEN-unset}"', + 'read_env_value "$1" APP_HOST >/dev/null; printf "%s" "${APP_TOKEN-unset}"', str(env), ) - self.assertEqual(result.stdout, "two|unset") + self.assertEqual(result.stdout, "unset") def test_bash_env_does_not_run_ahead_of_the_file(self) -> None: # A function shadows the builtin, so a hook that ran in the reading bash would answer for the file. From b83be3a159d9af0c18ddbb37090a56a41811dcf5 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 13:35:07 -0700 Subject: [PATCH 04/17] Share the Tree Ownership Rule Between carry.py and the Gate carry.py gains target_patterns and compare_declaration, which its own check and the carried-trees check both call, and its docstring states what the gate imports under plain python3. A carry.py that will not load or a manifest that will not read is now a note, the closing count names only compared trees, and the host-secrets check names which verdict takes the shared mode, as the governing rule does. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 35 +++++++++++++++----------- scripts/carry.py | 35 ++++++++++++++++++++------ spec/project-types.json | 2 +- tests/test_carry.py | 14 +++++++++++ tests/test_repo_gate.py | 16 ++++++++++++ 5 files changed, 78 insertions(+), 24 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index dcf61c3a4..87703c645 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -824,21 +824,27 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: if "spec/files.json" in files: NOTES.append("the scanned repository is a hub, whose trees are sources rather than copies.") return [] - carry = hub_carry(hub) + # Each failure is a note rather than a traceback, since one here would also skip every check sorted after this one. + try: + carry = hub_carry(hub) + except (ImportError, SyntaxError, OSError) as error: + NOTES.append( + f"the hub's scripts/carry.py could not be loaded, so no carried tree was compared: {error}" + ) + return [] + try: + trees = carry.load_json(hub / "spec" / "files.json")["trees"] + except carry.CarryError as error: + NOTES.append(f"no carried tree was compared: {error}") + return [] carried = 0 - for tree in carry.load_json(hub / "spec" / "files.json")["trees"]: - target, patterns = tree["target"], tree["include"] + for tree in trees: + target = tree["target"] prefix = target + "/" tracked_here = [path[len(prefix) :] for path in files if path.startswith(prefix)] if not tracked_here: continue - carried += 1 - # A pruned tree owns everything under its target, as a carry and the audit read it, so `include` narrows only one that does not prune. - owned = ( - tracked_here - if tree["prune"] - else [p for p in tracked_here if carry.included(p, patterns)] - ) + owned = [path for path in tracked_here if carry.included(path, carry.target_patterns(tree))] try: copy = { relative: None @@ -847,14 +853,13 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: else (root / target / relative).read_bytes() for relative in owned } - source = carry.inventory(hub / tree["source"], patterns) + source = carry.inventory(hub / tree["source"], tree["include"]) except (carry.CarryError, OSError) as error: NOTES.append(f"{target} could not be compared: {error}") continue - result = carry.compare(source, carry.Inventory(copy, frozenset(), "")) - changed, missing = result["modified"], result["missing"] - # A tree that does not prune leaves the repository's own files under it alone, as a carry does. - extra = result["extra"] if tree["prune"] else [] + carried += 1 + result = carry.compare_declaration(tree, source, carry.Inventory(copy, frozenset(), "")) + changed, missing, extra = result["modified"], result["missing"], result["extra"] if changed or missing or extra: if os.environ.get("GITHUB_ACTIONS") == "true": print( diff --git a/scripts/carry.py b/scripts/carry.py index 470185996..2c1baa9b1 100755 --- a/scripts/carry.py +++ b/scripts/carry.py @@ -3,7 +3,12 @@ # requires-python = ">=3.13" # dependencies = ["markdown-it-py>=4"] # /// -"""Check or apply manifest-owned trees, and the verbatim sections inside a mixed file, from the hub to a fleet worktree.""" +"""Check or apply manifest-owned trees, and the verbatim sections inside a mixed file, from the hub to a fleet worktree. + +The repository gate's `carried-trees` check loads this file under plain `python3`, 3.12 on a CI +runner, and calls `load_json`, `included`, `inventory`, `Inventory`, `CarryError`, +`target_patterns`, and `compare_declaration`. Those stay on the standard library and on 3.12. +""" import argparse import fnmatch @@ -141,6 +146,22 @@ def compare(source: Inventory, target: Inventory | None) -> dict[str, Any]: } +def target_patterns(declaration: dict[str, Any]) -> list[str]: + """What a declaration owns under its target: everything where it prunes, its `include` where it does not.""" + return ["**/*"] if declaration.get("prune") else declaration["include"] + + +def compare_declaration( + declaration: dict[str, Any], source: Inventory, target: Inventory | None +) -> dict[str, Any]: + """`compare`, with the extras cleared where the declaration does not prune, since it owns only what it carries there.""" + result = compare(source, target) + if not declaration.get("prune"): + result["extra"] = [] + result["extraDirectories"] = [] + return result + + def apply_tree( source: Inventory, target_root: pathlib.Path, @@ -890,12 +911,10 @@ def run(mode: str, name: str, target: pathlib.Path, hub: pathlib.Path = ROOT) -> source_root = relative_root(hub, declaration["source"]) target_root = relative_root(target, declaration["target"]) source = inventory(source_root, declaration["include"]) - target_patterns = ["**/*"] if declaration.get("prune") else declaration["include"] - target_inventory = inventory(target_root, target_patterns) if target_root.exists() else None - result = compare(source, target_inventory) - if not declaration.get("prune"): - result["extra"] = [] - result["extraDirectories"] = [] + target_inventory = ( + inventory(target_root, target_patterns(declaration)) if target_root.exists() else None + ) + result = compare_declaration(declaration, source, target_inventory) print( json.dumps( {"source": declaration["source"], "target": declaration["target"], **result}, @@ -919,7 +938,7 @@ def run(mode: str, name: str, target: pathlib.Path, hub: pathlib.Path = ROOT) -> if mode == "apply" and not declaration_clean: for change in apply_tree(source, target_root, target, result): print(change) - final = compare(source, inventory(target_root, target_patterns)) + final = compare(source, inventory(target_root, target_patterns(declaration))) print( json.dumps( { diff --git a/spec/project-types.json b/spec/project-types.json index 5eef339ef..d98ac851b 100644 --- a/spec/project-types.json +++ b/spec/project-types.json @@ -134,7 +134,7 @@ "host-secrets": { "detect": ["lib/fleet/sh/secrets.sh", "a shell script the repo runs that sources or reads a credential file under .secrets/ or ~/.secrets/"], "checks": [ - { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a credential from disk, a secret file or an env file holding one under .secrets/ or ~/.secrets/, sources the carried lib/fleet/sh/secrets.sh and checks the file with require_secret, secret_verdict, or secret_set_verdict before reading it, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file that sources the library, never in an edit to the carried copy, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#repo-scoped-secrets" } + { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a credential from disk, a secret file or an env file holding one under .secrets/ or ~/.secrets/, sources the carried lib/fleet/sh/secrets.sh and checks the file before reading it, with require_secret or secret_set_verdict for a private file, or secret_verdict with shared for one a container reads as a Docker secret, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file that sources the library, never in an edit to the carried copy, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#repo-scoped-secrets" } ] } }, diff --git a/tests/test_carry.py b/tests/test_carry.py index a7679713f..3e750e391 100755 --- a/tests/test_carry.py +++ b/tests/test_carry.py @@ -42,6 +42,20 @@ def test_compare_reports_missing_modified_extra_and_directories(self) -> None: self.assertEqual(result["missingDirectories"], ["empty"]) self.assertEqual(result["extraDirectories"], ["extra-dir"]) + def test_a_pruned_declaration_owns_everything_under_its_target(self) -> None: + self.assertEqual(carry.target_patterns({"prune": True, "include": ["*.txt"]}), ["**/*"]) + self.assertEqual(carry.target_patterns({"prune": False, "include": ["*.txt"]}), ["*.txt"]) + + def test_only_a_pruned_declaration_reports_extras(self) -> None: + source = carry.Inventory({"kept.txt": b"hub"}, frozenset(), "source") + target = carry.Inventory( + {"kept.txt": b"hub", "own.txt": b"repo"}, frozenset({"own-dir"}), "target" + ) + pruned = carry.compare_declaration({"prune": True}, source, target) + self.assertEqual((pruned["extra"], pruned["extraDirectories"]), (["own.txt"], ["own-dir"])) + kept = carry.compare_declaration({"prune": False}, source, target) + self.assertEqual((kept["extra"], kept["extraDirectories"]), ([], [])) + def test_empty_source_requires_target_root(self) -> None: with tempfile.TemporaryDirectory() as temp: root = pathlib.Path(temp) diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 4743dc26a..82d9718f8 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1448,6 +1448,22 @@ def test_a_tree_that_cannot_be_compared_is_a_note_and_never_a_finding(self) -> N files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] self.assertEqual(self.run_check(files), ([], "")) self.assertTrue(repo_gate.NOTES[0].startswith("lib/fleet/sh could not be compared: ")) + self.assertEqual( + repo_gate.NOTES[1], "compared 0 carried tree(s) against the hub tree beside this gate." + ) + + def test_a_carry_that_will_not_load_is_a_note_and_never_a_finding(self) -> None: + (self.hub / "scripts/carry.py").write_text("def broken(:\n", encoding="utf-8") + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] + self.assertEqual(self.run_check(files), ([], "")) + self.assertTrue( + repo_gate.NOTES[0].startswith("the hub's scripts/carry.py could not be loaded") + ) + + def test_an_unreadable_manifest_is_a_note_and_never_a_finding(self) -> None: + self.write(self.hub / "spec/files.json", "{") + self.assertEqual(self.run_check([]), ([], "")) + self.assertTrue(repo_gate.NOTES[0].startswith("no carried tree was compared: ")) def test_actions_shows_the_difference_on_the_pull_request(self) -> None: files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "edited\n")] From 0f40ab05feac2957012a95ccfa5f60491d785a14 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 13:43:03 -0700 Subject: [PATCH 05/17] Keep a Carry That Will Not Run From Failing the Gate The carried-trees check reports any failure to load or run the hub's carry.py as a note, since it runs under whatever python3 the host has, and carry.py states that contract in a comment rather than in its help text. carry.py's post-apply comparison reads a declaration's ownership as its check does. The rule text says the hub owns the carried directory whole, so a repository extends the library outside it. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 11 ++++++----- AUDIT.md | 2 +- GOVERNANCE.md | 4 ++-- lib/sh/secrets.sh | 2 +- scripts/README.md | 2 +- scripts/carry.py | 12 ++++++------ spec/project-types.json | 2 +- spec/type-model.md | 2 +- tests/test_repo_gate.py | 10 ++++++++++ 9 files changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 87703c645..ee5ace964 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -825,16 +825,17 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: NOTES.append("the scanned repository is a hub, whose trees are sources rather than copies.") return [] # Each failure is a note rather than a traceback, since one here would also skip every check sorted after this one. + # The catches are broad because carry.py runs under whatever `python3` the host has, so a newer interpreter's API in it fails here as any exception. try: carry = hub_carry(hub) - except (ImportError, SyntaxError, OSError) as error: + except Exception as error: # noqa: BLE001 NOTES.append( f"the hub's scripts/carry.py could not be loaded, so no carried tree was compared: {error}" ) return [] try: trees = carry.load_json(hub / "spec" / "files.json")["trees"] - except carry.CarryError as error: + except Exception as error: # noqa: BLE001 NOTES.append(f"no carried tree was compared: {error}") return [] carried = 0 @@ -844,8 +845,8 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: tracked_here = [path[len(prefix) :] for path in files if path.startswith(prefix)] if not tracked_here: continue - owned = [path for path in tracked_here if carry.included(path, carry.target_patterns(tree))] try: + owned = [p for p in tracked_here if carry.included(p, carry.target_patterns(tree))] copy = { relative: None if (root / target / relative).is_symlink() @@ -854,11 +855,11 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: for relative in owned } source = carry.inventory(hub / tree["source"], tree["include"]) - except (carry.CarryError, OSError) as error: + result = carry.compare_declaration(tree, source, carry.Inventory(copy, frozenset(), "")) + except Exception as error: # noqa: BLE001 NOTES.append(f"{target} could not be compared: {error}") continue carried += 1 - result = carry.compare_declaration(tree, source, carry.Inventory(copy, frozenset(), "")) changed, missing, extra = result["modified"], result["missing"], result["extra"] if changed or missing or extra: if os.environ.get("GITHUB_ACTIONS") == "true": diff --git a/AUDIT.md b/AUDIT.md index 98ff8b33d..9e18a647e 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -58,7 +58,7 @@ Otherwise read its `types[]`. If the entry is `classificationPending` (a backlog - `Dockerfile` + a docker build/push workflow -> `docker`, an `upstream-version.json` tracker -> `upstream-wrapper`. - `custom_components/*/manifest.json` + `hacs.json` -> `homeassistant`, a codegen workflow -> `codegen`, no `build-*` task -> `source-only`, governance-only -> `docs`. - `hugo.yaml` / `hugo.toml` / `config/_default/hugo.yaml` -> `hugo`. A repo may carry it alongside `source-only`, since a site deploy leaf is not a `build-*` task and both declarations stay true. -- a carried `lib/fleet/sh/secrets.sh`, or a shell script the repo runs that sources or reads a credential file under `.secrets/` or `~/.secrets/` -> `host-secrets`, which selects that carried tree. +- a carried `lib/fleet/sh/secrets.sh` -> `host-secrets`, and so does a shell script the repo runs that reads a credential file under `.secrets/` or `~/.secrets/`. ## 3. Applicability Gate diff --git a/GOVERNANCE.md b/GOVERNANCE.md index a4f96036f..85311a94c 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -123,7 +123,7 @@ A repo whose own stacks or scripts read local runtime credentials from disk, mos A fresh checkout then documents its own required shape without ever exposing a real value. This negation keeps a real secret file out of a **new** commit. It does not remove one already tracked: `.gitignore` has no effect on a path git already follows. A real secret file found tracked is removed from the index (`git rm --cached `) and its credential is rotated, not just added to `.gitignore` going forward. - **`.secrets/README.md` is a catalog**, one row per file naming what it holds and what consumes it, plus a short note on how the directory relates to `~/.secrets/` where the repo also touches that. -- **A repository whose shell scripts read a credential from disk declares the `host-secrets` type, and reads each one through the carried readers.** A credential is a secret file or an env file holding one, in `.secrets/` or in the host-scoped `~/.secrets/`, and this bullet reaches both. A single inline read counts, since it is the read a bare existence check and a sourced file get wrong. A script sources its carried `lib/fleet/sh/secrets.sh` and checks the file before reading it. A private file takes `require_secret` or `secret_set_verdict`, and one a container reads as a Docker secret takes `secret_verdict` with `shared`. It reads one value from an env file with `read_env_value` rather than sourcing the file into its own shell, per "Hub-Hosted Tooling". +- **A repository whose shell scripts read a credential from disk declares the `host-secrets` type, and reads each one through the carried readers.** A credential is a secret file or an env file holding one, in `.secrets/` or in the host-scoped `~/.secrets/`, and this bullet reaches both. A single inline read counts, since it is the read a bare existence check and a sourced file get wrong. A script sources its carried `lib/fleet/sh/secrets.sh`, per "Hub-Hosted Tooling", and checks the file before reading it. A private file takes `require_secret` or `secret_set_verdict`, and one a container reads as a Docker secret takes `secret_verdict` with `shared`. It reads one value from an env file with `read_env_value` rather than sourcing the file into its own shell. ## Repository Onboarding and Conformance @@ -140,7 +140,7 @@ The fleet's tooling lives in the hub once and a repository runs it from there ra **What a repository carries and what it reaches is decided by what the content is.** It carries the content it is audited against and the configuration that describes it, meaning its rule text and the files the manifest declares. It reaches machinery whose content is identical in every repository, meaning the prose and repository gates, the review digest, and the configuration script, because a file holding no per-repo content is a copy whose only future is to go stale. A tool named in a carried rule is therefore named as the hub's, since the alternative is a pointer to a path the reader does not have, and a pointer that resolves nowhere teaches the reader that a pointer in carried text is decorative. -**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root. Fetched from the hub at run time, it would let a GitHub outage or a bad `main` commit stop a scheduled host job. It would also run code the repository never reviewed against its own secrets. So offline operation and review in the repository that runs it outweigh the staleness of a copy. A gate differs, since "the tool did not run" is an acceptable answer there and CI is the backstop. The hub hosts the library once. A repository whose declared types select it carries it byte-for-byte, as a tree the hub's `spec/files.json` declares. It extends the library in a file of its own that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds. It reports a difference as a warning that never fails the gate. A failure would break every downstream pull request on each hub fix. The audit reports a stale copy, and the resync refreshes it. +**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root. Fetched from the hub at run time, it would let a GitHub outage or a bad `main` commit stop a scheduled host job. It would also run code the repository never reviewed against its own secrets. So offline operation and review in the repository that runs it outweigh the staleness of a copy. A gate differs, since "the tool did not run" is an acceptable answer there and CI is the backstop. The hub hosts the library once. A repository whose declared types select it carries it byte-for-byte, as a tree the hub's `spec/files.json` declares. The hub owns the carried directory whole, so the repository extends the library in a file of its own outside it that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds. It reports a difference as a warning that never fails the gate. A failure would break every downstream pull request on each hub fix. The audit reports a stale copy, and the resync refreshes it. **Reaching it is a checkout of the hub rather than a copy of one file.** A tool reads the payloads, tables, and sibling modules beside it, so a single file lifted out of the tree runs against whatever the caller happens to have, which is the copy problem again in a shorter loop. Read `main`, the promoted and gated state, and fetch immediately before running, because a clone is whatever it last fetched rather than the branch it names, and a stale clone answers confidently instead of failing. Name the tool by its path in that checkout and name the target explicitly, since a tool that defaults to the current directory or the current repository resolves somewhere either way, and a result computed against the wrong repository is well-formed. Which directory the command runs in is the tool's own contract rather than a rule here, so a gate reading a working tree runs in the repository under test while a tool taking its target as an argument runs anywhere and is given one. What the rule forbids is letting a default decide which repository the answer is about. diff --git a/lib/sh/secrets.sh b/lib/sh/secrets.sh index 3fb9c3d09..e48a5a0e7 100644 --- a/lib/sh/secrets.sh +++ b/lib/sh/secrets.sh @@ -3,7 +3,7 @@ # Meant to be sourced, never run: `. "$REPO/lib/fleet/sh/secrets.sh"` from a script that has already found its own tree through `BASH_SOURCE`. # It sets no shell option, since sourcing it would change the caller's own, and every function answers through its output or its status under any of them. # -# The hub hosts this file and a repository carries it byte-for-byte, per GOVERNANCE.md "Hub-Hosted Tooling", so a repository extends it in a file of its own that sources it, never by editing it. +# The hub hosts this file and a repository carries it byte-for-byte, per GOVERNANCE.md "Hub-Hosted Tooling", so a repository extends it in a file of its own outside the carried directory that sources it, never by editing it. # A caller that needs these functions in an elevated shell ships them there by name with `declare -f`, since sourcing this file in the calling shell does not reach one started by `sudo`. # A function here that calls another is shipped with it, so `secret_verdict` needs `secret_shape_verdict`, `secret_state`, `unresolved_link_reason`, `secret_content_verdict`, `secret_content_state`, and `secret_content_refusal`. diff --git a/scripts/README.md b/scripts/README.md index 6611c9b16..1bd993a55 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate, wherever the scanned repository tracks files at its target. The hub tree is the pinned action's in CI and the checkout's in a local run, and the comparison reuses [`carry.py`](#carrypy)'s inventory, so `include` and `prune` read as a carry reads them. A tree that cannot be compared, such as one whose hub source holds a link, says so in a note. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files is compared. The hub tree is the pinned action's in CI and the checkout's in a local run, and the comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared, such as one whose hub source holds a link, says so in a note, and so does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/scripts/carry.py b/scripts/carry.py index 2c1baa9b1..90b2f4c7b 100755 --- a/scripts/carry.py +++ b/scripts/carry.py @@ -3,12 +3,10 @@ # requires-python = ">=3.13" # dependencies = ["markdown-it-py>=4"] # /// -"""Check or apply manifest-owned trees, and the verbatim sections inside a mixed file, from the hub to a fleet worktree. +"""Check or apply manifest-owned trees, and the verbatim sections inside a mixed file, from the hub to a fleet worktree.""" -The repository gate's `carried-trees` check loads this file under plain `python3`, 3.12 on a CI -runner, and calls `load_json`, `included`, `inventory`, `Inventory`, `CarryError`, -`target_patterns`, and `compare_declaration`. Those stay on the standard library and on 3.12. -""" +# The repository gate's `carried-trees` check loads this whole file under plain `python3`, 3.12 on a CI runner, and calls its tree functions. +# The file's top level and those functions therefore stay on the standard library and on 3.12, and the gate reports a failure to load or run them as a note. import argparse import fnmatch @@ -938,7 +936,9 @@ def run(mode: str, name: str, target: pathlib.Path, hub: pathlib.Path = ROOT) -> if mode == "apply" and not declaration_clean: for change in apply_tree(source, target_root, target, result): print(change) - final = compare(source, inventory(target_root, target_patterns(declaration))) + final = compare_declaration( + declaration, source, inventory(target_root, target_patterns(declaration)) + ) print( json.dumps( { diff --git a/spec/project-types.json b/spec/project-types.json index d98ac851b..16454ef83 100644 --- a/spec/project-types.json +++ b/spec/project-types.json @@ -134,7 +134,7 @@ "host-secrets": { "detect": ["lib/fleet/sh/secrets.sh", "a shell script the repo runs that sources or reads a credential file under .secrets/ or ~/.secrets/"], "checks": [ - { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a credential from disk, a secret file or an env file holding one under .secrets/ or ~/.secrets/, sources the carried lib/fleet/sh/secrets.sh and checks the file before reading it, with require_secret or secret_set_verdict for a private file, or secret_verdict with shared for one a container reads as a Docker secret, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file that sources the library, never in an edit to the carried copy, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#repo-scoped-secrets" } + { "id": "hostsecrets.library.reader", "verdict": "intent", "assert": "Every shell script the repo runs that reads a credential from disk, a secret file or an env file holding one under .secrets/ or ~/.secrets/, sources the carried lib/fleet/sh/secrets.sh and checks the file before reading it, with require_secret or secret_set_verdict for a private file, or secret_verdict with shared for one a container reads as a Docker secret, and reads one value from an env file with read_env_value rather than sourcing the file into its own shell. A repo-specific helper lives in the repo's own file outside lib/fleet/sh that sources the library, never in an edit to the carried copy or a file beside it, whose byte identity the carried tree's verbatim-tree fidelity already checks.", "intentRef": "GOVERNANCE.md#repo-scoped-secrets" } ] } }, diff --git a/spec/type-model.md b/spec/type-model.md index 50feea612..336ded0d9 100644 --- a/spec/type-model.md +++ b/spec/type-model.md @@ -58,7 +58,7 @@ A generator type declares no `profiles`. Build versus lint-only is a depth of *l ## Runtime Libraries -A **runtime library** type declares that the repo's own runtime executes a library the hub hosts. The type's token selects the library's tree in [`spec/files.json`][files], and the repo carries it byte-for-byte. `host-secrets` is the one such type, for a repo whose shell scripts read a credential from disk, and it carries `lib/sh` from the hub to `lib/fleet/sh`. Its `detect` entry surfaces such a script as a discovery advisory, and the declaration is what makes the type's check apply, as for any type. A runtime library type declares no `profiles`, for the reason a generator type declares none. +A **runtime library** type declares that the repo's own runtime executes a library the hub hosts. The type's token selects the library's tree in [`spec/files.json`][files], and the repo carries it byte-for-byte. `host-secrets` is the one such type, for a repo whose shell scripts read a credential from disk. It carries `lib/sh` from the hub to `lib/fleet/sh`. Its `detect` entry surfaces such a script as a discovery advisory, and the declaration is what makes the type's check apply, as for any type. A runtime library type declares no `profiles`, for the reason a generator type declares none. ## Changing the type set carries review weight diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 82d9718f8..c95254e7b 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1460,6 +1460,16 @@ def test_a_carry_that_will_not_load_is_a_note_and_never_a_finding(self) -> None: repo_gate.NOTES[0].startswith("the hub's scripts/carry.py could not be loaded") ) + def test_a_carry_missing_a_function_the_gate_calls_is_a_note(self) -> None: + carry_file = self.hub / "scripts/carry.py" + text = carry_file.read_text(encoding="utf-8") + carry_file.write_text( + text.replace("def compare_declaration(", "def renamed("), encoding="utf-8" + ) + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] + self.assertEqual(self.run_check(files), ([], "")) + self.assertTrue(repo_gate.NOTES[0].startswith("lib/fleet/sh could not be compared: ")) + def test_an_unreadable_manifest_is_a_note_and_never_a_finding(self) -> None: self.write(self.hub / "spec/files.json", "{") self.assertEqual(self.run_check([]), ([], "")) From a1fda9f139cce894c5c6c72e30e94555fa2cb759 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 13:55:56 -0700 Subject: [PATCH 06/17] Prove the Gate Loads carry.py Under Python 3.12 A case runs the carried-trees check under a 3.12 interpreter, since a CI runner's system python3 is 3.12 and hub CI otherwise runs 3.13 alone. A case drives carry.py apply through its post-apply comparison for a tree that does not prune. The gate cases that need a colon in a file name or a symlink no longer fail on Windows, and the README gives carried-trees its own reason for a note. Co-Authored-By: Claude Opus 5.5 --- scripts/README.md | 2 +- tests/test_carry.py | 37 +++++++++++++++++++++++++++++++++++++ tests/test_repo_gate.py | 37 ++++++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 1bd993a55..6538e14e9 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -154,7 +154,7 @@ The resolution pass is **scoped to the scanned repository's own owner**, which i A pin is a finding only where GitHub **answered** that the commit is absent, meaning a 404 or a 422. An offline host, a missing `gh`, a rate limit and a narrow token all report as unread and leave the pin on its shape, so the gate stays usable with no network instead of failing a correct tree. A 404 is confirmed against the repository itself before it becomes a finding, since an absent commit and a repository the credentials cannot see are the same answer from here, and a repository-scoped CI token is narrower than the fleet. That second read runs only on the failing path, and each distinct pin is read once however often it repeats. -A `note:` line is how a check says it did less than its name, or, for `carried-trees`, how it reports a difference that is deliberately not a finding. It prints under the check's own result, outside the issue count, and never changes the exit code, because nothing is wrong with the tree when the network is what is missing. It carries one fixed shape every run so a zero is as visible as a count, and it separates a pin under a known other owner from one whose owner could not be compared at all, since a checkout with no readable `origin` skips every pin including this owner's own and reporting those as somebody else's is the same false clean the note was added to prevent. +A `note:` line is how a check says it did less than its name. `carried-trees` also uses one to report a difference it deliberately does not count, for the reason its bullet above gives. It prints under the check's own result, outside the issue count, and never changes the exit code. For a check that read less, that is because nothing is wrong with the tree when the network is what is missing. It carries one fixed shape every run so a zero is as visible as a count, and it separates a pin under a known other owner from one whose owner could not be compared at all, since a checkout with no readable `origin` skips every pin including this owner's own and reporting those as somebody else's is the same false clean the note was added to prevent. A stale-backticked-path check was built and **rejected**: a template repo legitimately references paths that live in downstream repos, so it produced 34 false positives on a clean tree with no way to separate those from real drift. Doc-to-doc drift is a review lens, not a regex. diff --git a/tests/test_carry.py b/tests/test_carry.py index 3e750e391..f31c62f38 100755 --- a/tests/test_carry.py +++ b/tests/test_carry.py @@ -3,6 +3,9 @@ from __future__ import annotations +import contextlib +import io +import json import pathlib import re import subprocess @@ -42,6 +45,40 @@ def test_compare_reports_missing_modified_extra_and_directories(self) -> None: self.assertEqual(result["missingDirectories"], ["empty"]) self.assertEqual(result["extraDirectories"], ["extra-dir"]) + def test_apply_leaves_a_repository_file_under_a_tree_that_does_not_prune(self) -> None: + """The post-apply comparison reads ownership as the check does, so it accepts what apply kept.""" + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + hub = root / "hub" + (hub / "src").mkdir(parents=True) + (hub / "src" / "carried.txt").write_text("hub", encoding="utf-8") + (hub / "registry").mkdir() + (hub / "registry" / "repos.json").write_text("{}", encoding="utf-8") + (hub / "spec").mkdir() + tree = { + "source": "src", + "target": "dst", + "fidelity": "verbatim-tree", + "appliesTo": "*", + "include": ["**/*"], + "prune": False, + } + (hub / "spec" / "files.json").write_text( + json.dumps({"trees": [tree]}), encoding="utf-8" + ) + target = root / "repo" + (target / "dst").mkdir(parents=True) + (target / "dst" / "own.txt").write_text("mine", encoding="utf-8") + with ( + mock.patch.object(carry, "verify_hub", return_value="0" * 40), + mock.patch.object(carry, "resolve_repo", return_value={"types": ["docs"]}), + mock.patch.object(carry, "verify_target"), + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(carry.run("apply", "Demo", target, hub), 0) + self.assertEqual((target / "dst" / "carried.txt").read_text(encoding="utf-8"), "hub") + self.assertEqual((target / "dst" / "own.txt").read_text(encoding="utf-8"), "mine") + def test_a_pruned_declaration_owns_everything_under_its_target(self) -> None: self.assertEqual(carry.target_patterns({"prune": True, "include": ["*.txt"]}), ["**/*"]) self.assertEqual(carry.target_patterns({"prune": False, "include": ["*.txt"]}), ["*.txt"]) diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index c95254e7b..e83068658 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -28,6 +28,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / ".github/actions/repo-gate")) import repo_gate +from host_capability import requires_symlink from runner_commands import FORGING_LINES, FORGING_NAMES, ORDINARY_LINES, forged_lines REPO = Path(__file__).resolve().parent.parent @@ -1414,6 +1415,7 @@ def test_an_untracked_file_is_not_a_difference(self) -> None: self.run_check(files) self.assertEqual(len(repo_gate.NOTES), 1) + @requires_symlink def test_a_link_in_the_copy_is_a_change(self) -> None: (self.repo / "lib/fleet/sh").mkdir(parents=True) (self.repo / "lib/fleet/sh/secrets.sh").symlink_to(self.hub / "lib/sh/secrets.sh") @@ -1443,6 +1445,7 @@ def test_a_narrowed_include_narrows_both_sides_of_a_tree_that_does_not_prune(sel self.narrowed(prune=False) self.assertEqual(len(repo_gate.NOTES), 1) + @requires_symlink def test_a_tree_that_cannot_be_compared_is_a_note_and_never_a_finding(self) -> None: (self.hub / "lib/sh/link").symlink_to(self.hub / "lib/sh/secrets.sh") files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] @@ -1460,6 +1463,37 @@ def test_a_carry_that_will_not_load_is_a_note_and_never_a_finding(self) -> None: repo_gate.NOTES[0].startswith("the hub's scripts/carry.py could not be loaded") ) + @unittest.skipUnless(shutil.which("uv"), "uv provides the 3.12 interpreter") + def test_the_hub_carry_runs_under_the_runners_python_3_12(self) -> None: + """A CI runner's system python3 is 3.12, and hub CI otherwise loads carry.py under 3.13 alone.""" + repo = self.tmp / "scanned" + self.write(repo / "lib/fleet/sh/secrets.sh", "stale\n") + subprocess.run(["git", "init", "-q", str(repo)], check=True, timeout=60) + subprocess.run(["git", "-C", str(repo), "add", "-A"], check=True, timeout=60) + gate = REPO / ".github/actions/repo-gate/repo_gate.py" + result = subprocess.run( + [ + "uv", + "run", + "--no-project", + "--python", + "3.12", + "python", + "-I", + str(gate), + "--root", + str(repo), + "--check", + "carried-trees", + ], + capture_output=True, + encoding="utf-8", + timeout=300, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("lib/fleet/sh differs from the hub's lib/sh: 1 changed", result.stdout) + def test_a_carry_missing_a_function_the_gate_calls_is_a_note(self) -> None: carry_file = self.hub / "scripts/carry.py" text = carry_file.read_text(encoding="utf-8") @@ -1483,7 +1517,8 @@ def test_actions_shows_the_difference_on_the_pull_request(self) -> None: ) def test_the_annotation_carries_no_name_from_the_scanned_repository(self) -> None: - files = [self.write(self.repo / "lib/fleet/sh/x%0A::error::forged", "")] + # Named in the tracked list alone, since a Windows file name cannot hold `:`, and a tracked name with no file reads as changed. + files = ["lib/fleet/sh/x%0A::error::forged"] _, printed = self.run_check(files, actions=True) self.assertEqual(printed.count("\n"), 1) self.assertNotIn("forged", printed) From 9007a2edbe03d77c91e97038b6f02ac23c77c069 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 14:04:34 -0700 Subject: [PATCH 07/17] Run the 3.12 Case Only Where 3.12 Is Installed The case that loads carry.py under Python 3.12 now skips where uv finds no 3.12 already installed, and never downloads one. Co-Authored-By: Claude Opus 5.5 --- GOVERNANCE.md | 2 +- scripts/README.md | 2 +- tests/test_repo_gate.py | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 85311a94c..c8596b41f 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -140,7 +140,7 @@ The fleet's tooling lives in the hub once and a repository runs it from there ra **What a repository carries and what it reaches is decided by what the content is.** It carries the content it is audited against and the configuration that describes it, meaning its rule text and the files the manifest declares. It reaches machinery whose content is identical in every repository, meaning the prose and repository gates, the review digest, and the configuration script, because a file holding no per-repo content is a copy whose only future is to go stale. A tool named in a carried rule is therefore named as the hub's, since the alternative is a pointer to a path the reader does not have, and a pointer that resolves nowhere teaches the reader that a pointer in carried text is decorative. -**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root. Fetched from the hub at run time, it would let a GitHub outage or a bad `main` commit stop a scheduled host job. It would also run code the repository never reviewed against its own secrets. So offline operation and review in the repository that runs it outweigh the staleness of a copy. A gate differs, since "the tool did not run" is an acceptable answer there and CI is the backstop. The hub hosts the library once. A repository whose declared types select it carries it byte-for-byte, as a tree the hub's `spec/files.json` declares. The hub owns the carried directory whole, so the repository extends the library in a file of its own outside it that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds. It reports a difference as a warning that never fails the gate. A failure would break every downstream pull request on each hub fix. The audit reports a stale copy, and the resync refreshes it. +**Code a repository's own runtime executes is the exception, and it is carried.** Such a library runs on a host the project owns, often unattended and sometimes as root. Fetched from the hub at run time, it would let a GitHub outage or a bad `main` commit stop a scheduled host job. It would also run code the repository never reviewed against its own secrets. So offline operation and review in the repository that runs it outweigh the staleness of a copy. A gate differs, since "the tool did not run" is an acceptable answer there and CI is the backstop. The hub hosts the library once. A repository whose declared types select it carries it byte-for-byte, as a tree the hub's `spec/files.json` declares. The hub owns the carried directory whole. The repository extends the library in a file of its own outside it that sources it, never by editing the copy. The staleness is made visible rather than accepted. The repository gate compares each carried tree with the hub tree its pinned action holds. It reports a difference as a warning that never fails the gate. A failure would break every downstream pull request on each hub fix. The audit reports a stale copy, and the resync refreshes it. **Reaching it is a checkout of the hub rather than a copy of one file.** A tool reads the payloads, tables, and sibling modules beside it, so a single file lifted out of the tree runs against whatever the caller happens to have, which is the copy problem again in a shorter loop. Read `main`, the promoted and gated state, and fetch immediately before running, because a clone is whatever it last fetched rather than the branch it names, and a stale clone answers confidently instead of failing. Name the tool by its path in that checkout and name the target explicitly, since a tool that defaults to the current directory or the current repository resolves somewhere either way, and a result computed against the wrong repository is well-formed. Which directory the command runs in is the tool's own contract rather than a rule here, so a gate reading a working tree runs in the repository under test while a tool taking its target as an argument runs anywhere and is given one. What the rule forbids is letting a default decide which repository the answer is about. diff --git a/scripts/README.md b/scripts/README.md index 6538e14e9..d45eb704c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files is compared. The hub tree is the pinned action's in CI and the checkout's in a local run, and the comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared, such as one whose hub source holds a link, says so in a note, and so does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files is compared. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index e83068658..3a45324ea 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1339,6 +1339,19 @@ def test_a_schema_violation_fails(self) -> None: self.assertTrue(any(h.startswith("schema:") for h in hits)) +def python_3_12_installed() -> bool: + """Whether uv finds a 3.12 already installed, so the case below never downloads one.""" + if shutil.which("uv") is None: + return False + found = subprocess.run( + ["uv", "python", "find", "--no-python-downloads", "3.12"], + capture_output=True, + timeout=60, + check=False, + ) + return found.returncode == 0 + + class TestCarriedTrees(TreeCase): """A constructed hub beside the gate and a constructed copy, so each case proves one difference.""" @@ -1463,7 +1476,7 @@ def test_a_carry_that_will_not_load_is_a_note_and_never_a_finding(self) -> None: repo_gate.NOTES[0].startswith("the hub's scripts/carry.py could not be loaded") ) - @unittest.skipUnless(shutil.which("uv"), "uv provides the 3.12 interpreter") + @unittest.skipUnless(python_3_12_installed(), "no Python 3.12 is installed for uv to run") def test_the_hub_carry_runs_under_the_runners_python_3_12(self) -> None: """A CI runner's system python3 is 3.12, and hub CI otherwise loads carry.py under 3.13 alone.""" repo = self.tmp / "scanned" @@ -1476,6 +1489,7 @@ def test_the_hub_carry_runs_under_the_runners_python_3_12(self) -> None: "uv", "run", "--no-project", + "--no-python-downloads", "--python", "3.12", "python", From 185a41e79cfa144815989dfa4b0c4e9bef95aeca Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 14:15:51 -0700 Subject: [PATCH 08/17] Count a Tree as Carried Only From the Files It Owns A tree that does not prune shares its target with the repository's own files, so those alone no longer make the gate compare it and report every hub file missing. The 3.12 skip moves to host_capability beside the suite's other host checks, and the README names the sha-pin note as the subject of its last sentences. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 3 +++ scripts/README.md | 2 +- tests/host_capability.py | 19 +++++++++++++++++++ tests/test_repo_gate.py | 25 ++++++++++--------------- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index ee5ace964..f9fc4f4b1 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -847,6 +847,9 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: continue try: owned = [p for p in tracked_here if carry.included(p, carry.target_patterns(tree))] + # A tree that does not prune shares its target with the repository's own files, so those alone do not make it carried. + if not owned: + continue copy = { relative: None if (root / target / relative).is_symlink() diff --git a/scripts/README.md b/scripts/README.md index d45eb704c..c45ccb5ce 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -154,7 +154,7 @@ The resolution pass is **scoped to the scanned repository's own owner**, which i A pin is a finding only where GitHub **answered** that the commit is absent, meaning a 404 or a 422. An offline host, a missing `gh`, a rate limit and a narrow token all report as unread and leave the pin on its shape, so the gate stays usable with no network instead of failing a correct tree. A 404 is confirmed against the repository itself before it becomes a finding, since an absent commit and a repository the credentials cannot see are the same answer from here, and a repository-scoped CI token is narrower than the fleet. That second read runs only on the failing path, and each distinct pin is read once however often it repeats. -A `note:` line is how a check says it did less than its name. `carried-trees` also uses one to report a difference it deliberately does not count, for the reason its bullet above gives. It prints under the check's own result, outside the issue count, and never changes the exit code. For a check that read less, that is because nothing is wrong with the tree when the network is what is missing. It carries one fixed shape every run so a zero is as visible as a count, and it separates a pin under a known other owner from one whose owner could not be compared at all, since a checkout with no readable `origin` skips every pin including this owner's own and reporting those as somebody else's is the same false clean the note was added to prevent. +A `note:` line is how a check says it did less than its name. `carried-trees` also uses one to report a difference it deliberately does not count, for the reason its bullet above gives. It prints under the check's own result, outside the issue count, and never changes the exit code. For a check that read less, that is because nothing is wrong with the tree when the network is what is missing. The `sha-pin` note carries one fixed shape every run so a zero is as visible as a count. It separates a pin under a known other owner from one whose owner could not be compared at all. A checkout with no readable `origin` skips every pin, this owner's own included, and reporting those as somebody else's is the same false clean the note was added to prevent. A stale-backticked-path check was built and **rejected**: a template repo legitimately references paths that live in downstream repos, so it produced 34 false positives on a clean tree with no way to separate those from real drift. Doc-to-doc drift is a review lens, not a regex. diff --git a/tests/host_capability.py b/tests/host_capability.py index e54041791..a80936a94 100644 --- a/tests/host_capability.py +++ b/tests/host_capability.py @@ -35,6 +35,25 @@ def can_symlink() -> bool: requires_symlink = unittest.skipUnless(can_symlink(), NO_SYMLINK) +@functools.cache +def python_3_12_installed() -> bool: + """Whether uv finds a Python 3.12 already installed, so a case using it never downloads one.""" + if shutil.which("uv") is None: + return False + found = subprocess.run( + ["uv", "python", "find", "--no-python-downloads", "3.12"], + capture_output=True, + timeout=60, + check=False, + ) + return found.returncode == 0 + + +requires_python_3_12 = unittest.skipUnless( + python_3_12_installed(), "no Python 3.12 is installed for uv to run" +) + + @functools.cache def unprivileged_root_is_available() -> bool: """Whether `unshare -r` gives this process an EUID of 0 without any real privilege. diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 3a45324ea..68a32ea77 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -28,7 +28,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / ".github/actions/repo-gate")) import repo_gate -from host_capability import requires_symlink +from host_capability import requires_python_3_12, requires_symlink from runner_commands import FORGING_LINES, FORGING_NAMES, ORDINARY_LINES, forged_lines REPO = Path(__file__).resolve().parent.parent @@ -1339,19 +1339,6 @@ def test_a_schema_violation_fails(self) -> None: self.assertTrue(any(h.startswith("schema:") for h in hits)) -def python_3_12_installed() -> bool: - """Whether uv finds a 3.12 already installed, so the case below never downloads one.""" - if shutil.which("uv") is None: - return False - found = subprocess.run( - ["uv", "python", "find", "--no-python-downloads", "3.12"], - capture_output=True, - timeout=60, - check=False, - ) - return found.returncode == 0 - - class TestCarriedTrees(TreeCase): """A constructed hub beside the gate and a constructed copy, so each case proves one difference.""" @@ -1422,6 +1409,14 @@ def test_a_tree_that_does_not_prune_leaves_a_repository_file_alone(self) -> None self.run_check(files) self.assertEqual(len(repo_gate.NOTES), 1) + def test_only_files_a_tree_that_does_not_prune_owns_make_it_carried(self) -> None: + self.manifest([self.tree(include=["fleet_*.sh"], prune=False)]) + self.write(self.hub / "lib/sh/fleet_secrets.sh", "hub\n") + self.run_check([self.write(self.repo / "lib/fleet/sh/own.sh", "mine\n")]) + self.assertEqual( + repo_gate.NOTES, ["compared 0 carried tree(s) against the hub tree beside this gate."] + ) + def test_an_untracked_file_is_not_a_difference(self) -> None: files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] self.write(self.repo / "lib/fleet/sh/scratch.sh", "untracked\n") @@ -1476,7 +1471,7 @@ def test_a_carry_that_will_not_load_is_a_note_and_never_a_finding(self) -> None: repo_gate.NOTES[0].startswith("the hub's scripts/carry.py could not be loaded") ) - @unittest.skipUnless(python_3_12_installed(), "no Python 3.12 is installed for uv to run") + @requires_python_3_12 def test_the_hub_carry_runs_under_the_runners_python_3_12(self) -> None: """A CI runner's system python3 is 3.12, and hub CI otherwise loads carry.py under 3.13 alone.""" repo = self.tmp / "scanned" From 9b35692f1aa5b5fb2c7edade368e28909e6ce843 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 14:25:49 -0700 Subject: [PATCH 09/17] Judge a Link in a Carried Copy Without a Placeholder The gate keeps a tracked link or non-file out of the inventory it hands carry.py and classifies it itself, changed where the hub carries that path and extra under a pruned tree where it does not. The 3.12 probe catches a uv that fails or hangs, as the probe beside it does, and the README says which tracked files make a tree compared. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 17 +++++++++++++---- scripts/README.md | 2 +- tests/host_capability.py | 15 +++++++++------ tests/test_repo_gate.py | 7 +++++++ 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index f9fc4f4b1..ccc03ddca 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -850,12 +850,17 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: # A tree that does not prune shares its target with the repository's own files, so those alone do not make it carried. if not owned: continue - copy = { - relative: None + # A tracked link, or a path that is not a regular file, holds no content to compare, so it is kept out of the inventory and judged apart. + irregular = { + relative + for relative in owned if (root / target / relative).is_symlink() or not (root / target / relative).is_file() - else (root / target / relative).read_bytes() + } + copy = { + relative: (root / target / relative).read_bytes() for relative in owned + if relative not in irregular } source = carry.inventory(hub / tree["source"], tree["include"]) result = carry.compare_declaration(tree, source, carry.Inventory(copy, frozenset(), "")) @@ -863,7 +868,11 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: NOTES.append(f"{target} could not be compared: {error}") continue carried += 1 - changed, missing, extra = result["modified"], result["missing"], result["extra"] + changed = sorted({*result["modified"], *(irregular & source.files.keys())}) + missing = [path for path in result["missing"] if path not in irregular] + extra = sorted( + {*result["extra"], *(irregular - source.files.keys() if tree["prune"] else ())} + ) if changed or missing or extra: if os.environ.get("GITHUB_ACTIONS") == "true": print( diff --git a/scripts/README.md b/scripts/README.md index c45ccb5ce..b6852d752 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files is compared. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared, every file under the target for a tree that prunes and the files its `include` matches for one that does not. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/tests/host_capability.py b/tests/host_capability.py index a80936a94..5e9fb221d 100644 --- a/tests/host_capability.py +++ b/tests/host_capability.py @@ -40,12 +40,15 @@ def python_3_12_installed() -> bool: """Whether uv finds a Python 3.12 already installed, so a case using it never downloads one.""" if shutil.which("uv") is None: return False - found = subprocess.run( - ["uv", "python", "find", "--no-python-downloads", "3.12"], - capture_output=True, - timeout=60, - check=False, - ) + try: + found = subprocess.run( + ["uv", "python", "find", "--no-python-downloads", "3.12"], + capture_output=True, + timeout=60, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return False return found.returncode == 0 diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 68a32ea77..6e0d9d985 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1430,6 +1430,13 @@ def test_a_link_in_the_copy_is_a_change(self) -> None: self.run_check(["lib/fleet/sh/secrets.sh"]) self.assertIn("1 changed", repo_gate.NOTES[0]) + @requires_symlink + def test_a_link_the_hub_does_not_carry_is_extra_under_a_pruned_tree(self) -> None: + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] + (self.repo / "lib/fleet/sh/other.sh").symlink_to(self.hub / "lib/sh/secrets.sh") + self.run_check([*files, "lib/fleet/sh/other.sh"]) + self.assertIn("0 changed, 0 missing, 1 extra, first other.sh.", repo_gate.NOTES[0]) + def test_a_tree_the_repository_does_not_carry_is_not_compared(self) -> None: self.run_check([self.write(self.repo / "README.md", "")]) self.assertEqual( From e1b2fa164307628e617aaebe0926a25bfedf6c56 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 14:36:11 -0700 Subject: [PATCH 10/17] Name a Carried Target Git Tracks as a Link A target, or an ancestor of it, that Git tracks as a link or a file now names itself in a note instead of reading as a tree the repository does not carry. A tracked file deleted from the working tree reads as missing rather than changed. The gate cases pin their full note strings and cover a link under a tree that does not prune. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 14 ++++++++++-- scripts/README.md | 4 ++-- tests/test_repo_gate.py | 30 ++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index ccc03ddca..038e772ad 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -842,6 +842,13 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: for tree in trees: target = tree["target"] prefix = target + "/" + # Git tracks a link to a directory as one path, so a target or an ancestor tracked that way hides every file beneath it. + blocking = next((path for path in files if prefix.startswith(path + "/")), None) + if blocking is not None: + NOTES.append( + f"{target} was not compared, since {blocking} is tracked as a link or a file." + ) + continue tracked_here = [path[len(prefix) :] for path in files if path.startswith(prefix)] if not tracked_here: continue @@ -851,16 +858,19 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: if not owned: continue # A tracked link, or a path that is not a regular file, holds no content to compare, so it is kept out of the inventory and judged apart. + # A tracked file deleted from the working tree is left out as well, so it reads as missing. irregular = { relative for relative in owned if (root / target / relative).is_symlink() - or not (root / target / relative).is_file() + or ( + (root / target / relative).exists() and not (root / target / relative).is_file() + ) } copy = { relative: (root / target / relative).read_bytes() for relative in owned - if relative not in irregular + if relative not in irregular and (root / target / relative).is_file() } source = carry.inventory(hub / tree["source"], tree["include"]) result = carry.compare_declaration(tree, source, carry.Inventory(copy, frozenset(), "")) diff --git a/scripts/README.md b/scripts/README.md index b6852d752..b50151fcb 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared, every file under the target for a tree that prunes and the files its `include` matches for one that does not. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py @@ -154,7 +154,7 @@ The resolution pass is **scoped to the scanned repository's own owner**, which i A pin is a finding only where GitHub **answered** that the commit is absent, meaning a 404 or a 422. An offline host, a missing `gh`, a rate limit and a narrow token all report as unread and leave the pin on its shape, so the gate stays usable with no network instead of failing a correct tree. A 404 is confirmed against the repository itself before it becomes a finding, since an absent commit and a repository the credentials cannot see are the same answer from here, and a repository-scoped CI token is narrower than the fleet. That second read runs only on the failing path, and each distinct pin is read once however often it repeats. -A `note:` line is how a check says it did less than its name. `carried-trees` also uses one to report a difference it deliberately does not count, for the reason its bullet above gives. It prints under the check's own result, outside the issue count, and never changes the exit code. For a check that read less, that is because nothing is wrong with the tree when the network is what is missing. The `sha-pin` note carries one fixed shape every run so a zero is as visible as a count. It separates a pin under a known other owner from one whose owner could not be compared at all. A checkout with no readable `origin` skips every pin, this owner's own included, and reporting those as somebody else's is the same false clean the note was added to prevent. +A `note:` line is how a check says it did less than its name. `carried-trees` also uses one to report a difference it deliberately does not count, for the reason its bullet above gives. It prints under the check's own result, outside the issue count, and never changes the exit code. For a check that read less, that is because nothing is wrong with the tree when the network is what is missing. The `sha-pin` note carries one fixed shape every run so a zero is as visible as a count. It separates a pin under a known other owner from one whose owner could not be compared at all. A checkout with no readable `origin` skips every pin, this owner's own included. Reporting those as somebody else's is the same false clean the note was added to prevent. A stale-backticked-path check was built and **rejected**: a template repo legitimately references paths that live in downstream repos, so it produced 34 false positives on a clean tree with no way to separate those from real drift. Doc-to-doc drift is a review lens, not a regex. diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 6e0d9d985..2b3eaf26a 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1428,7 +1428,32 @@ def test_a_link_in_the_copy_is_a_change(self) -> None: (self.repo / "lib/fleet/sh").mkdir(parents=True) (self.repo / "lib/fleet/sh/secrets.sh").symlink_to(self.hub / "lib/sh/secrets.sh") self.run_check(["lib/fleet/sh/secrets.sh"]) - self.assertIn("1 changed", repo_gate.NOTES[0]) + self.assertIn("1 changed, 0 missing, 0 extra, first secrets.sh.", repo_gate.NOTES[0]) + + @requires_symlink + def test_a_link_the_hub_does_not_carry_is_left_alone_by_a_tree_that_does_not_prune( + self, + ) -> None: + self.manifest([self.tree(prune=False)]) + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] + (self.repo / "lib/fleet/sh/other.sh").symlink_to(self.hub / "lib/sh/secrets.sh") + self.run_check([*files, "lib/fleet/sh/other.sh"]) + self.assertEqual(len(repo_gate.NOTES), 1) + + def test_a_tracked_file_deleted_from_the_working_tree_is_missing(self) -> None: + (self.repo / "lib/fleet/sh").mkdir(parents=True) + self.run_check(["lib/fleet/sh/secrets.sh"]) + self.assertIn("0 changed, 1 missing, 0 extra, first secrets.sh.", repo_gate.NOTES[0]) + + def test_a_target_tracked_as_a_link_is_named_rather_than_skipped(self) -> None: + for tracked in ("lib/fleet/sh", "lib/fleet"): + with self.subTest(tracked=tracked): + repo_gate.NOTES.clear() + self.assertEqual(self.run_check([tracked]), ([], "")) + self.assertEqual( + repo_gate.NOTES[0], + f"lib/fleet/sh was not compared, since {tracked} is tracked as a link or a file.", + ) @requires_symlink def test_a_link_the_hub_does_not_carry_is_extra_under_a_pruned_tree(self) -> None: @@ -1533,7 +1558,8 @@ def test_actions_shows_the_difference_on_the_pull_request(self) -> None: ) def test_the_annotation_carries_no_name_from_the_scanned_repository(self) -> None: - # Named in the tracked list alone, since a Windows file name cannot hold `:`, and a tracked name with no file reads as changed. + # Named in the tracked list alone, since a Windows file name cannot hold `:`. + # The hub's own file then reads as missing. files = ["lib/fleet/sh/x%0A::error::forged"] _, printed = self.run_check(files, actions=True) self.assertEqual(printed.count("\n"), 1) From fc347871530a2fe8747b254710551360c607d20d Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 14:49:21 -0700 Subject: [PATCH 11/17] Pin the Carried-Trees Summary Notes in Their Tests The cases that expect no difference now assert the whole note list, so a tree the check skips no longer passes for one it compared. The gate keeps only tracked links apart, since a checkout produces no other non-file at a tracked path, and the README documents the note for a target Git tracks as a link or a file. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 13 +++---------- scripts/README.md | 2 +- tests/test_repo_gate.py | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 038e772ad..f37cde031 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -857,16 +857,9 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: # A tree that does not prune shares its target with the repository's own files, so those alone do not make it carried. if not owned: continue - # A tracked link, or a path that is not a regular file, holds no content to compare, so it is kept out of the inventory and judged apart. - # A tracked file deleted from the working tree is left out as well, so it reads as missing. - irregular = { - relative - for relative in owned - if (root / target / relative).is_symlink() - or ( - (root / target / relative).exists() and not (root / target / relative).is_file() - ) - } + # A tracked link holds no content of its own to compare, so it is kept out of the inventory and judged apart. + # A tracked file absent from the working tree is left out as well, so it reads as missing. + irregular = {relative for relative in owned if (root / target / relative).is_symlink()} copy = { relative: (root / target / relative).read_bytes() for relative in owned diff --git a/scripts/README.md b/scripts/README.md index b50151fcb..d619be6d3 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note, since Git then lists nothing beneath it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 2b3eaf26a..3ce1632df 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1407,7 +1407,9 @@ def test_a_tree_that_does_not_prune_leaves_a_repository_file_alone(self) -> None self.write(self.repo / "lib/fleet/sh/local.sh", "mine\n"), ] self.run_check(files) - self.assertEqual(len(repo_gate.NOTES), 1) + self.assertEqual( + repo_gate.NOTES, ["compared 1 carried tree(s) against the hub tree beside this gate."] + ) def test_only_files_a_tree_that_does_not_prune_owns_make_it_carried(self) -> None: self.manifest([self.tree(include=["fleet_*.sh"], prune=False)]) @@ -1421,7 +1423,9 @@ def test_an_untracked_file_is_not_a_difference(self) -> None: files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] self.write(self.repo / "lib/fleet/sh/scratch.sh", "untracked\n") self.run_check(files) - self.assertEqual(len(repo_gate.NOTES), 1) + self.assertEqual( + repo_gate.NOTES, ["compared 1 carried tree(s) against the hub tree beside this gate."] + ) @requires_symlink def test_a_link_in_the_copy_is_a_change(self) -> None: @@ -1438,7 +1442,9 @@ def test_a_link_the_hub_does_not_carry_is_left_alone_by_a_tree_that_does_not_pru files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] (self.repo / "lib/fleet/sh/other.sh").symlink_to(self.hub / "lib/sh/secrets.sh") self.run_check([*files, "lib/fleet/sh/other.sh"]) - self.assertEqual(len(repo_gate.NOTES), 1) + self.assertEqual( + repo_gate.NOTES, ["compared 1 carried tree(s) against the hub tree beside this gate."] + ) def test_a_tracked_file_deleted_from_the_working_tree_is_missing(self) -> None: (self.repo / "lib/fleet/sh").mkdir(parents=True) @@ -1483,7 +1489,9 @@ def test_a_narrowed_include_narrows_the_hub_side_only_of_a_pruned_tree(self) -> def test_a_narrowed_include_narrows_both_sides_of_a_tree_that_does_not_prune(self) -> None: self.narrowed(prune=False) - self.assertEqual(len(repo_gate.NOTES), 1) + self.assertEqual( + repo_gate.NOTES, ["compared 1 carried tree(s) against the hub tree beside this gate."] + ) @requires_symlink def test_a_tree_that_cannot_be_compared_is_a_note_and_never_a_finding(self) -> None: From 1b59d2ffde2f96e0043e902fd26bd007bf922ea7 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 15:00:20 -0700 Subject: [PATCH 12/17] Refuse a Link in a Carried Copy as a Carry Does carry.py refuses a symlink in a tree, so the gate now reports a tracked link under a carried target as a tree it could not compare, naming the link, instead of classifying it by a prune rule of its own. The ownership rule then lives in compare_declaration alone. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 19 +++++------ scripts/README.md | 2 +- tests/test_repo_gate.py | 44 ++++++++++++-------------- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index f37cde031..2e3b0fbec 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -857,13 +857,18 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: # A tree that does not prune shares its target with the repository's own files, so those alone do not make it carried. if not owned: continue - # A tracked link holds no content of its own to compare, so it is kept out of the inventory and judged apart. - # A tracked file absent from the working tree is left out as well, so it reads as missing. - irregular = {relative for relative in owned if (root / target / relative).is_symlink()} + # A carry refuses a link in a tree, so the gate refuses one too rather than judging it by a rule of its own. + link = next((p for p in sorted(owned) if (root / target / p).is_symlink()), None) + if link is not None: + NOTES.append( + f"{target} could not be compared, since {link} is a link, which a carry refuses." + ) + continue + # A tracked file absent from the working tree is left out, so it reads as missing. copy = { relative: (root / target / relative).read_bytes() for relative in owned - if relative not in irregular and (root / target / relative).is_file() + if (root / target / relative).is_file() } source = carry.inventory(hub / tree["source"], tree["include"]) result = carry.compare_declaration(tree, source, carry.Inventory(copy, frozenset(), "")) @@ -871,11 +876,7 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: NOTES.append(f"{target} could not be compared: {error}") continue carried += 1 - changed = sorted({*result["modified"], *(irregular & source.files.keys())}) - missing = [path for path in result["missing"] if path not in irregular] - extra = sorted( - {*result["extra"], *(irregular - source.files.keys() if tree["prune"] else ())} - ) + changed, missing, extra = result["modified"], result["missing"], result["extra"] if changed or missing or extra: if os.environ.get("GITHUB_ACTIONS") == "true": print( diff --git a/scripts/README.md b/scripts/README.md index d619be6d3..de352900a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note, since Git then lists nothing beneath it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note. Git then lists nothing beneath it. A link inside a carried tree is refused in a note, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 3ce1632df..5269e9db2 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1428,23 +1428,26 @@ def test_an_untracked_file_is_not_a_difference(self) -> None: ) @requires_symlink - def test_a_link_in_the_copy_is_a_change(self) -> None: - (self.repo / "lib/fleet/sh").mkdir(parents=True) - (self.repo / "lib/fleet/sh/secrets.sh").symlink_to(self.hub / "lib/sh/secrets.sh") - self.run_check(["lib/fleet/sh/secrets.sh"]) - self.assertIn("1 changed, 0 missing, 0 extra, first secrets.sh.", repo_gate.NOTES[0]) - - @requires_symlink - def test_a_link_the_hub_does_not_carry_is_left_alone_by_a_tree_that_does_not_prune( - self, - ) -> None: - self.manifest([self.tree(prune=False)]) - files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] - (self.repo / "lib/fleet/sh/other.sh").symlink_to(self.hub / "lib/sh/secrets.sh") - self.run_check([*files, "lib/fleet/sh/other.sh"]) - self.assertEqual( - repo_gate.NOTES, ["compared 1 carried tree(s) against the hub tree beside this gate."] - ) + def test_a_link_in_the_copy_is_refused_as_a_carry_refuses_it(self) -> None: + for prune in (True, False): + with self.subTest(prune=prune): + repo_gate.NOTES.clear() + self.manifest([self.tree(prune=prune)]) + (self.repo / "lib/fleet/sh").mkdir(parents=True, exist_ok=True) + link = self.repo / "lib/fleet/sh/secrets.sh" + link.unlink(missing_ok=True) + link.symlink_to(self.hub / "lib/sh/secrets.sh") + self.assertEqual(self.run_check(["lib/fleet/sh/secrets.sh"]), ([], "")) + self.assertEqual( + repo_gate.NOTES, + [ + ( + "lib/fleet/sh could not be compared, since secrets.sh is a link, " + "which a carry refuses." + ), + "compared 0 carried tree(s) against the hub tree beside this gate.", + ], + ) def test_a_tracked_file_deleted_from_the_working_tree_is_missing(self) -> None: (self.repo / "lib/fleet/sh").mkdir(parents=True) @@ -1461,13 +1464,6 @@ def test_a_target_tracked_as_a_link_is_named_rather_than_skipped(self) -> None: f"lib/fleet/sh was not compared, since {tracked} is tracked as a link or a file.", ) - @requires_symlink - def test_a_link_the_hub_does_not_carry_is_extra_under_a_pruned_tree(self) -> None: - files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] - (self.repo / "lib/fleet/sh/other.sh").symlink_to(self.hub / "lib/sh/secrets.sh") - self.run_check([*files, "lib/fleet/sh/other.sh"]) - self.assertIn("0 changed, 0 missing, 1 extra, first other.sh.", repo_gate.NOTES[0]) - def test_a_tree_the_repository_does_not_carry_is_not_compared(self) -> None: self.run_check([self.write(self.repo / "README.md", "")]) self.assertEqual( From b0e2cf191b26f7d9ac990e50f699a15ace4ced70 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 15:09:34 -0700 Subject: [PATCH 13/17] Warn on a Link Anywhere Under a Carried Target The gate refuses a link anywhere under a carried target, as carry.inventory does, not only among the files the tree owns, and shows the refusal on the pull request as a warning beside its note. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 16 ++++++++++------ scripts/README.md | 2 +- tests/test_repo_gate.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 2e3b0fbec..3df8879d1 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -799,6 +799,12 @@ def hub_carry(hub: Path) -> ModuleType: return module +def warn_carried_tree(message: str) -> None: + """Show `message` on the pull request under Actions, built from manifest values alone so nothing a scanned repository names reaches the command.""" + if os.environ.get("GITHUB_ACTIONS") == "true": + print(f"::warning title=carried-trees::{message}") + + def check_carried_trees(root: Path, files: list[str]) -> list[str]: """Report each manifest-owned tree the scanned repository carries that differs from the hub's. @@ -857,9 +863,10 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: # A tree that does not prune shares its target with the repository's own files, so those alone do not make it carried. if not owned: continue - # A carry refuses a link in a tree, so the gate refuses one too rather than judging it by a rule of its own. - link = next((p for p in sorted(owned) if (root / target / p).is_symlink()), None) + # A carry refuses a link anywhere under a tree's target, so the gate refuses one too rather than judging it by a rule of its own. + link = next((p for p in sorted(tracked_here) if (root / target / p).is_symlink()), None) if link is not None: + warn_carried_tree(f"{target} holds a link, which a carry refuses") NOTES.append( f"{target} could not be compared, since {link} is a link, which a carry refuses." ) @@ -878,10 +885,7 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: carried += 1 changed, missing, extra = result["modified"], result["missing"], result["extra"] if changed or missing or extra: - if os.environ.get("GITHUB_ACTIONS") == "true": - print( - f"::warning title=carried-trees::{target} differs from the hub's {tree['source']}" - ) + warn_carried_tree(f"{target} differs from the hub's {tree['source']}") NOTES.append( f"{target} differs from the hub's {tree['source']}: {len(changed)} changed, " f"{len(missing)} missing, {len(extra)} extra, first " diff --git a/scripts/README.md b/scripts/README.md index de352900a..17bd21459 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note. Git then lists nothing beneath it. A link inside a carried tree is refused in a note, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note. Git then lists nothing beneath it. A link anywhere under a carried target is refused in a note and a warning, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 5269e9db2..8369f54e2 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1428,6 +1428,20 @@ def test_an_untracked_file_is_not_a_difference(self) -> None: ) @requires_symlink + @requires_symlink + def test_a_link_the_tree_does_not_own_is_refused_as_well(self) -> None: + self.manifest([self.tree(include=["*.sh"], prune=False)]) + files = [self.write(self.repo / "lib/fleet/sh/secrets.sh", "hub\n")] + (self.repo / "lib/fleet/sh/notes.txt").symlink_to(self.hub / "lib/sh/secrets.sh") + _, printed = self.run_check([*files, "lib/fleet/sh/notes.txt"], actions=True) + self.assertEqual( + printed, + "::warning title=carried-trees::lib/fleet/sh holds a link, which a carry refuses\n", + ) + self.assertTrue( + repo_gate.NOTES[0].startswith("lib/fleet/sh could not be compared, since notes.txt") + ) + def test_a_link_in_the_copy_is_refused_as_a_carry_refuses_it(self) -> None: for prune in (True, False): with self.subTest(prune=prune): From f22ccc6a3ed831fe6b83ec8d63ee2c5f07a5607f Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 15:15:34 -0700 Subject: [PATCH 14/17] Restore the Symlink Guard on the Link Refusal Case The link refusal case gets back the symlink guard the previous commit moved onto its new neighbor, and the README says an exclude pattern covering part of a carried target reads the files it hides as missing. Co-Authored-By: Claude Opus 5.5 --- scripts/README.md | 2 +- tests/test_repo_gate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 17bd21459..a37929c00 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note. Git then lists nothing beneath it. A link anywhere under a carried target is refused in a note and a warning, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. An `--exclude` pattern narrows this scan like the others, so one covering part of a carried target reads the files it hides as missing. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note. Git then lists nothing beneath it. A link anywhere under a carried target is refused in a note and a warning, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 8369f54e2..0309996ff 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1427,7 +1427,6 @@ def test_an_untracked_file_is_not_a_difference(self) -> None: repo_gate.NOTES, ["compared 1 carried tree(s) against the hub tree beside this gate."] ) - @requires_symlink @requires_symlink def test_a_link_the_tree_does_not_own_is_refused_as_well(self) -> None: self.manifest([self.tree(include=["*.sh"], prune=False)]) @@ -1442,6 +1441,7 @@ def test_a_link_the_tree_does_not_own_is_refused_as_well(self) -> None: repo_gate.NOTES[0].startswith("lib/fleet/sh could not be compared, since notes.txt") ) + @requires_symlink def test_a_link_in_the_copy_is_refused_as_a_carry_refuses_it(self) -> None: for prune in (True, False): with self.subTest(prune=prune): From c2a3e003f7e423e2fb4b704d1cb51a487687a730 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 15:26:06 -0700 Subject: [PATCH 15/17] Warn When a Carried Target Is Itself a Link A carried target that Git tracks as a link or a file now warns on the pull request as well as naming itself in a note, since a carry refuses it. An ancestor tracked that way stays a note, because a tracked file at a generic path says nothing about a carry. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 3 +++ scripts/README.md | 2 +- tests/test_repo_gate.py | 8 ++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 3df8879d1..5433cd0d8 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -851,6 +851,9 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: # Git tracks a link to a directory as one path, so a target or an ancestor tracked that way hides every file beneath it. blocking = next((path for path in files if prefix.startswith(path + "/")), None) if blocking is not None: + # Only the target itself warns, since a tracked file at a generic ancestor such as `lib` says nothing about a carry. + if blocking == target: + warn_carried_tree(f"{target} is tracked as a link or a file, which a carry refuses") NOTES.append( f"{target} was not compared, since {blocking} is tracked as a link or a file." ) diff --git a/scripts/README.md b/scripts/README.md index a37929c00..ced5549e2 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. An `--exclude` pattern narrows this scan like the others, so one covering part of a carried target reads the files it hides as missing. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note. Git then lists nothing beneath it. A link anywhere under a carried target is refused in a note and a warning, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. An `--exclude` pattern narrows this scan like the others, so one covering part of a carried target reads the files it hides as missing. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note, and the target itself also warns. Git then lists nothing beneath it. A link anywhere under a carried target is refused in a note and a warning, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 0309996ff..3f7760df1 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1469,10 +1469,14 @@ def test_a_tracked_file_deleted_from_the_working_tree_is_missing(self) -> None: self.assertIn("0 changed, 1 missing, 0 extra, first secrets.sh.", repo_gate.NOTES[0]) def test_a_target_tracked_as_a_link_is_named_rather_than_skipped(self) -> None: - for tracked in ("lib/fleet/sh", "lib/fleet"): + warning = ( + "::warning title=carried-trees::lib/fleet/sh is tracked as a link or a file, " + "which a carry refuses\n" + ) + for tracked, printed in (("lib/fleet/sh", warning), ("lib/fleet", "")): with self.subTest(tracked=tracked): repo_gate.NOTES.clear() - self.assertEqual(self.run_check([tracked]), ([], "")) + self.assertEqual(self.run_check([tracked], actions=True), ([], printed)) self.assertEqual( repo_gate.NOTES[0], f"lib/fleet/sh was not compared, since {tracked} is tracked as a link or a file.", From 15f6363406092b2423a2995c453b89cec1ee947a Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 15:36:11 -0700 Subject: [PATCH 16/17] Resolve the Apply Case Temp Root and Pass Over Generic Paths The carry.py apply case resolves its temp root as main resolves a target, since a Windows temp path can be an 8.3 short name, which failed the Windows unit test job. A carried target or a non-top-level ancestor tracked as a link or a file now warns as well as notes, and a generic top-level path tracked that way is passed over like an uncarried tree. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 11 +++++++---- scripts/README.md | 2 +- tests/test_carry.py | 3 ++- tests/test_repo_gate.py | 12 +++++++++--- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 5433cd0d8..14b3b6ef9 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -849,11 +849,14 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: target = tree["target"] prefix = target + "/" # Git tracks a link to a directory as one path, so a target or an ancestor tracked that way hides every file beneath it. - blocking = next((path for path in files if prefix.startswith(path + "/")), None) + # A top-level path such as `lib` is generic, so one tracked that way says nothing about a carry and is passed over like an uncarried tree. + blocking = next( + (path for path in files if "/" in path and prefix.startswith(path + "/")), None + ) if blocking is not None: - # Only the target itself warns, since a tracked file at a generic ancestor such as `lib` says nothing about a carry. - if blocking == target: - warn_carried_tree(f"{target} is tracked as a link or a file, which a carry refuses") + warn_carried_tree( + f"{target} sits at or below a tracked link or file, which a carry refuses" + ) NOTES.append( f"{target} was not compared, since {blocking} is tracked as a link or a file." ) diff --git a/scripts/README.md b/scripts/README.md index ced5549e2..40c8a91aa 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -123,7 +123,7 @@ Deterministic checks: - `eol`: every path pinned LF in [`.gitattributes`][gitattributes] has the matching [`.editorconfig`][editorconfig] override the line-ending rule requires, with EditorConfig brace syntax expanded. One direction only: an `.editorconfig` LF glob with no git pin is legitimate, since `.editorconfig` governs what the editor writes where git enforces a class it must not guess at. - `eol-coverage`: the same pins read against the tree instead. A tracked file opening `#!` that git does not resolve to `eol=lf` is an interpreter line a CRLF checkout breaks, and a pin matching no tracked file is dead unless its block is marked `forward-declared`. - `composite-actions`: every tracked `action.yml` or `action.yaml` under `.github/actions/`, at any depth, is shellchecked and schema-checked, because actionlint reads workflows only. Each non-empty `run:` body of a bash, sh, or unset-shell step goes to shellcheck. Every `${{ }}` expression is replaced by a placeholder first. A finding names the action file and the step. A pwsh or python step is skipped. The file set goes to `check-jsonschema`'s `vendor.github-actions` builtin through `uvx check-jsonschema@latest`, or `pipx run` where `uvx` is absent. Shellcheck runs from the `mirror.gcr.io/koalaman/shellcheck:stable` image, so the check needs Docker. It reads the action through PyYAML, in process where it is importable and through `uvx --with pyyaml` otherwise. Each half reports its own missing tool. Bodies run at warning severity without SC2154, so this is weaker than shellcheck over a `.sh` file. A tree with no action file is skipped with a note. -- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. An `--exclude` pattern narrows this scan like the others, so one covering part of a carried target reads the files it hides as missing. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A target, or an ancestor of it, that Git tracks as a link or a file says so in a note, and the target itself also warns. Git then lists nothing beneath it. A link anywhere under a carried target is refused in a note and a warning, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. +- `carried-trees`: each tree [`spec/files.json`][files] declares is compared byte-for-byte with the hub tree beside the gate. Only a tree whose target holds tracked files the tree owns is compared. A tree that prunes owns every file under its target, and one that does not owns the files its `include` matches. The hub tree is the pinned action's in CI and the checkout's in a local run. The comparison reuses [`carry.py`](#carrypy)'s tree functions, so `include` and `prune` read as a carry reads them. A tree that cannot be compared says so in a note, such as one whose hub source holds a link. So does a `carry.py` that will not load. A difference is a `note:` naming its counts and first path, plus a `::warning` annotation on Actions. It is never a finding, so a hub fix does not fail every downstream pull request. A byte difference cannot tell a stale copy from an edited one, so the audit tells them apart and the resync applies the refresh. In CI the comparison is with the pin, so a copy refreshed from a newer hub `main` also warns until Dependabot advances the pin. An `--exclude` pattern narrows this scan like the others, so one covering part of a carried target reads the files it hides as missing. Which trees a repository owes is the audit's, read from its declared types, so a tree the repository does not track is not compared. A note and a warning name a target tracked by Git as a link or a file. So does such an ancestor below the top level. Git then lists nothing beneath it. A link anywhere under a carried target is refused in a note and a warning, as a carry refuses it. A single-file fetch has no hub tree beside it, and says so in a note. A repository tracking `spec/files.json` is a hub, so it compares nothing, since its trees are the sources. ```sh python3 scripts/repo_gate.py diff --git a/tests/test_carry.py b/tests/test_carry.py index f31c62f38..1fd97b870 100755 --- a/tests/test_carry.py +++ b/tests/test_carry.py @@ -48,7 +48,8 @@ def test_compare_reports_missing_modified_extra_and_directories(self) -> None: def test_apply_leaves_a_repository_file_under_a_tree_that_does_not_prune(self) -> None: """The post-apply comparison reads ownership as the check does, so it accepts what apply kept.""" with tempfile.TemporaryDirectory() as temp: - root = pathlib.Path(temp) + # Resolved as `main` resolves its target, since a Windows temp path can be an 8.3 short name that `run` resolves to its long form. + root = pathlib.Path(temp).resolve() hub = root / "hub" (hub / "src").mkdir(parents=True) (hub / "src" / "carried.txt").write_text("hub", encoding="utf-8") diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 3f7760df1..430dc8e67 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1470,18 +1470,24 @@ def test_a_tracked_file_deleted_from_the_working_tree_is_missing(self) -> None: def test_a_target_tracked_as_a_link_is_named_rather_than_skipped(self) -> None: warning = ( - "::warning title=carried-trees::lib/fleet/sh is tracked as a link or a file, " + "::warning title=carried-trees::lib/fleet/sh sits at or below a tracked link or file, " "which a carry refuses\n" ) - for tracked, printed in (("lib/fleet/sh", warning), ("lib/fleet", "")): + for tracked in ("lib/fleet/sh", "lib/fleet"): with self.subTest(tracked=tracked): repo_gate.NOTES.clear() - self.assertEqual(self.run_check([tracked], actions=True), ([], printed)) + self.assertEqual(self.run_check([tracked], actions=True), ([], warning)) self.assertEqual( repo_gate.NOTES[0], f"lib/fleet/sh was not compared, since {tracked} is tracked as a link or a file.", ) + def test_a_generic_top_level_path_tracked_as_a_file_is_passed_over(self) -> None: + self.assertEqual(self.run_check(["lib"], actions=True), ([], "")) + self.assertEqual( + repo_gate.NOTES, ["compared 0 carried tree(s) against the hub tree beside this gate."] + ) + def test_a_tree_the_repository_does_not_carry_is_not_compared(self) -> None: self.run_check([self.write(self.repo / "README.md", "")]) self.assertEqual( From 85b2c4adb01fae676f9df876b33b3e2dc3b0e85d Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 26 Oct 2026 15:44:05 -0700 Subject: [PATCH 17/17] Exempt Only a Generic Ancestor, Never the Target The generic top-level exemption now applies to an ancestor alone, so a tree whose target is top-level still warns when Git tracks that target as a link or a file, and a case pins it. Co-Authored-By: Claude Opus 5.5 --- .github/actions/repo-gate/repo_gate.py | 9 +++++++-- tests/test_repo_gate.py | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 14b3b6ef9..3177b6c3e 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -849,9 +849,14 @@ def check_carried_trees(root: Path, files: list[str]) -> list[str]: target = tree["target"] prefix = target + "/" # Git tracks a link to a directory as one path, so a target or an ancestor tracked that way hides every file beneath it. - # A top-level path such as `lib` is generic, so one tracked that way says nothing about a carry and is passed over like an uncarried tree. + # A top-level ancestor such as `lib` is generic, so one tracked that way says nothing about a carry and is passed over like an uncarried tree. blocking = next( - (path for path in files if "/" in path and prefix.startswith(path + "/")), None + ( + path + for path in files + if (path == target or "/" in path) and prefix.startswith(path + "/") + ), + None, ) if blocking is not None: warn_carried_tree( diff --git a/tests/test_repo_gate.py b/tests/test_repo_gate.py index 430dc8e67..0c133e7ae 100755 --- a/tests/test_repo_gate.py +++ b/tests/test_repo_gate.py @@ -1482,6 +1482,11 @@ def test_a_target_tracked_as_a_link_is_named_rather_than_skipped(self) -> None: f"lib/fleet/sh was not compared, since {tracked} is tracked as a link or a file.", ) + def test_a_top_level_target_tracked_as_a_link_still_warns(self) -> None: + self.manifest([{**self.tree(), "target": "tools"}]) + printed = "::warning title=carried-trees::tools sits at or below a tracked link or file, which a carry refuses\n" + self.assertEqual(self.run_check(["tools"], actions=True), ([], printed)) + def test_a_generic_top_level_path_tracked_as_a_file_is_passed_over(self) -> None: self.assertEqual(self.run_check(["lib"], actions=True), ([], "")) self.assertEqual(