From 0ddd61618aa026586c22fcb5cd09a325b2025d17 Mon Sep 17 00:00:00 2001 From: sepehr-safari Date: Sat, 12 Sep 2026 21:50:28 +0300 Subject: [PATCH] The macOS installer checks the file it downloaded Installing Notary on macOS fails with a checksum mismatch right now. The download is not the problem: the installer compares it against a different file. installer expects: 61c71dc5... (notary-0.10.11-linux-aarch64.tar.gz) the macOS zip is: 2ca2053b... (correct) The digest came from `grep -o 'sha256:...' | head -1` over the whole release response, which takes the first digest in the JSON whatever asset it belongs to. Correct while a release carried one asset, and wrong from the moment the Linux tarballs were added: GitHub lists assets in upload order, so the first digest belongs to whichever packaging job finished first. Plaza has the same bug and hit it at v0.19.0, where macOS happened to lose that race. Notary's is not intermittent: its Linux job has been winning, so this has been broken on every macOS install since the Linux tarballs landed. That is the worst way for a checksum to fail. The bytes are right and the comparison is against something else, so the tool tells people their download is corrupt and refuses. A check that cries wolf teaches people to skip checks. The digest now comes from the asset that names the file. The response is pretty-printed, so it is flattened BEFORE being split on the `{` that starts each object: without the flatten every field is already on its own line and the name and the digest can never meet, which is a way of getting this wrong that looks like it works when tested against the wrong sample. The guard is the same one Plaza now carries, as a script rather than shell buried in a workflow so it can be run by hand. A fixture with a Linux asset listed first, because that is the case that breaks and it has to break deterministically. The live release, because a fixture only encodes what I think the response looks like and what can break this again is GitHub changing exactly that. And that the shipped script is the one those checks describe. This reaches people the moment it merges. The installer is fetched from main, not from a release, so no version bump is involved. --- .github/workflows/ci.yml | 13 ++++++ scripts/check-macos-installer.sh | 70 ++++++++++++++++++++++++++++++++ scripts/install-macos.sh | 26 +++++++++++- 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100755 scripts/check-macos-installer.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 632c6fe..87e802f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,19 @@ jobs: fi echo "release notes are for v$version" + # The installer verifies the file it downloaded, not whatever is listed + # first. The script read the first `sha256:` in the release JSON, which + # was correct while a release carried one asset and became wrong the + # moment the Linux tarballs were added: GitHub lists assets in upload + # order, so the first digest belongs to whichever packaging job finished + # first. On v0.10.11 that is the Linux aarch64 tarball, so every macOS + # install failed with a checksum mismatch on a perfectly good download. + # + # The checks live in the script so they can be run by hand, and so this + # file does not carry shell that only ever runs on a runner. + - name: The macOS installer verifies the macOS download + run: scripts/check-macos-installer.sh ${{ github.repository }} . + # The GUI (Native SDK): macOS only, via the native CLI. gui: runs-on: macos-latest diff --git a/scripts/check-macos-installer.sh b/scripts/check-macos-installer.sh new file mode 100755 index 0000000..b7379d4 --- /dev/null +++ b/scripts/check-macos-installer.sh @@ -0,0 +1,70 @@ +# The installer verifies the file it downloaded, not whatever is listed first. +# +# Two checks, because they catch different things. +set -eu +repo="$1" +prefix="$2" + +extract() { printf '%s' "$1" | tr -d '\n' | tr '{' '\n' | grep 'macos\.zip' | grep -o 'sha256:[0-9a-f]\{64\}' | head -1 | cut -d: -f2 || true; } + +# 1. A FIXTURE with a Linux asset first. This is the case that broke, and it +# breaks deterministically whether or not the live release happens to be +# ordered that way today. Pretty-printed and carrying an `uploader` object, +# because the real response is both and a flat fixture would pass against +# code that cannot read the real thing. +fixture='{ + "tag_name": "v9.9.9", + "assets": [ + { + "name": "notary-9.9.9-linux-aarch64.tar.gz", + "uploader": { "login": "github-actions[bot]", "id": 41898282 }, + "digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "browser_download_url": "https://example.com/notary-9.9.9-linux-aarch64.tar.gz" + }, + { + "name": "Notary-v9.9.9-macos.zip", + "uploader": { "login": "github-actions[bot]", "id": 41898282 }, + "digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "browser_download_url": "https://example.com/Notary-v9.9.9-macos.zip" + } + ] +}' +got="$(extract "$fixture")" +if [ "$got" != "2222222222222222222222222222222222222222222222222222222222222222" ]; then + echo "With a Linux asset listed first, the installer picks the wrong digest." + echo " picked: ${got:-nothing}" + echo " wanted: 2222... (the macOS zip)" + echo "That is the same failure Plaza hit at v0.19.0: a correct download reported as corrupt." + exit 1 +fi +echo "ok: a Linux asset listed first does not steal the macOS digest" + +# 2. The LIVE release, because a fixture only encodes what I think the response +# looks like, and what can break this again is GitHub changing exactly that. +json="$(curl -fsSL "https://api.github.com/repos/$repo/releases/latest")" +want="$(printf '%s' "$json" | python3 -c 'import json,sys +d=json.load(sys.stdin) +for a in d.get("assets",[]): + if a["name"].endswith("macos.zip"): + print((a.get("digest") or "").replace("sha256:","")) + break')" +if [ -z "$want" ]; then + echo "ok: the latest release publishes no macOS digest, nothing to compare" +else + got="$(extract "$json")" + if [ "$got" != "$want" ]; then + echo "Against the live release, the installer picks the wrong digest." + echo " picked: ${got:-nothing}" + echo " wanted: $want" + exit 1 + fi + echo "ok: against the live release the installer picks the macOS digest ($want)" +fi + +# 3. And the script that ships is the one these checks describe. +grep -q "tr -d '\\\\n' | tr '{' " "$prefix/scripts/install-macos.sh" || { + echo "$prefix/scripts/install-macos.sh no longer flattens before splitting." + exit 1 +} +bash -n "$prefix/scripts/install-macos.sh" +echo "ok: the shipped script is the one that was checked" diff --git a/scripts/install-macos.sh b/scripts/install-macos.sh index cdd72bb..33360ca 100755 --- a/scripts/install-macos.sh +++ b/scripts/install-macos.sh @@ -42,7 +42,31 @@ main() { local tag url digest tag="$(printf '%s' "$json" | grep -o '"tag_name":[[:space:]]*"[^"]*"' | head -1 | sed -E 's/.*"([^"]+)".*/\1/')" url="$(printf '%s' "$json" | grep -o '"browser_download_url":[[:space:]]*"[^"]*macos\.zip"' | head -1 | sed -E 's/.*"(https[^"]+)".*/\1/')" - digest="$(printf '%s' "$json" | grep -o 'sha256:[0-9a-f]\{64\}' | head -1 | cut -d: -f2)" + # The digest OF THE FILE BEING DOWNLOADED, which is not the same thing as the + # first digest in the release. + # + # This used to be `grep -o 'sha256:...' | head -1` over the whole response, + # which takes the first digest in the JSON whatever asset it belongs to. + # Correct while a release carried one asset, and wrong from the moment the + # Linux tarballs were added: GitHub lists assets in upload order, so the first + # digest belongs to whichever packaging job finished first. On v0.10.11 that + # is notary-0.10.11-linux-aarch64.tar.gz, so every macOS install fails with a + # checksum mismatch on a download that is perfectly fine. + # + # That is the worst way for a checksum to fail. The bytes are right and the + # comparison is against something else, so the tool tells people their + # download is corrupt and refuses. A check that cries wolf teaches people to + # skip checks. + # + # So the digest is read from the asset that names the file. The response is + # pretty-printed, so it is FLATTENED FIRST and only then split on the `{` that + # starts each object: without the flatten every field is already on its own + # line and the name and the digest can never meet. The segment naming the + # macOS zip ends at the next asset's `{`, so it cannot reach a neighbour's. + # + # No jq. This runs before anything is installed and uses only what macOS + # already ships. Plaza carries the identical fix and the identical guard. + digest="$(printf '%s' "$json" | tr -d '\n' | tr '{' '\n' | grep 'macos\.zip' | grep -o 'sha256:[0-9a-f]\{64\}' | head -1 | cut -d: -f2 || true)" [ -n "$url" ] || die "no macOS build found on the latest release (${tag:-unknown})." say "Latest release: ${tag:-unknown}"