From 1d988dd350c8e9043b9d27b091555b3eab51140d Mon Sep 17 00:00:00 2001 From: AminDhouib Date: Sat, 12 Sep 2026 04:01:40 -0400 Subject: [PATCH 1/3] fix(mc): fall back to the archived minio/mc GitHub release assets and verify every download against a pinned sha256 (dl.min.io answers 410) dl.min.io has returned "410 Gone -- the MinIO Client project is archived" for every mc release since 2026-09-11/12, and the org mirror at storage.devino.ca was never populated (404), so both existing sources are dead and every consumer job fails at "Install mc" (postify run 34680054913, lighthouse job). Add the release assets of the archived github.com/minio/mc repo as a third source after the two existing ones, and pin the per-platform sha256 of the binaries so any source can be checked before install; a mismatch is recorded and skipped rather than installed. The failure now names all three URLs with their own errors instead of only the last. Verified live on all five platforms. --- README.md | 15 +++- lib/main.py | 55 +++++++++++- tests/test_ensure_mc.py | 187 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+), 6 deletions(-) create mode 100644 tests/test_ensure_mc.py diff --git a/README.md b/README.md index 17467d8..3003dee 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,19 @@ to GitHub's blob store. This action writes to devino instead. MinIO STS (`AssumeRoleWithWebIdentity`) for one-hour credentials. MinIO picks the policy named after the token's `repository_owner_id` claim, so only workflows owned by DevinoSolutions get access; tokens from any other owner map to no policy and are refused. -2. A pinned `mc` (MinIO client) is fetched from `storage.devino.ca/tools/` - (fallback: dl.min.io) and cached in the runner tool cache. +2. A pinned `mc` (MinIO client) is fetched from `storage.devino.ca/tools/` and + cached in the runner tool cache. Two fallbacks follow the org mirror: + `dl.min.io`, and the release assets of the archived `github.com/minio/mc` + repository. Whichever source answers, the bytes are checked against the + per-platform sha256 pinned in `lib/main.py` (`MC_SHA256`) before the binary + is installed; a mismatch is never installed and the next source is tried. + If every source fails, the error names each URL with its own reason. + + `dl.min.io` has answered `410 Gone` since 2026-09-11/12 — MinIO archived the + client and stopped serving those files — so today the GitHub release assets + are the only source that responds, and the mirror should be populated at + `storage.devino.ca/tools/mc///mc`. Re-pin `MC_SHA256` + from each release's `.sha256sum` asset whenever `MC_VERSION` changes. 3. Upload: matched files are packed into one `.tgz` whose root mirrors upstream semantics (a single directory uploads its contents; several paths share their least common ancestor), then copied to diff --git a/lib/main.py b/lib/main.py index 5d7b6f6..8c33639 100644 --- a/lib/main.py +++ b/lib/main.py @@ -30,6 +30,24 @@ import urllib.request MC_VERSION = "RELEASE.2025-08-13T08-35-41Z" +# sha256 of the pinned `mc` binary for each supported platform. Every download, +# from whichever source, is checked against this table before it is installed: +# the primary source is an org-controlled bucket and the last resort is an +# archived third-party repository, so the pin is what makes the three sources +# interchangeable instead of three different trust levels. +# +# Taken on 2026-09-12 from the `.sha256sum` asset published next to each binary +# on the archived upstream release +# https://github.com/minio/mc/releases/tag/RELEASE.2025-08-13T08-35-41Z +# (e.g. mc.linux-amd64.RELEASE.2025-08-13T08-35-41Z.sha256sum). Re-pin these +# whenever MC_VERSION changes. +MC_SHA256 = { + "linux-amd64": "01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891", + "linux-arm64": "14c8c9616cfce4636add161304353244e8de383b2e2752c0e9dad01d4c27c12c", + "darwin-amd64": "2862c79cce11b09be9a8911a279b2e9465bebf74b9f01abca9c348a0d795f0cb", + "darwin-arm64": "a877fd0c183409da9f20f9d6e1811987298bbbca1aa03428eebdffba79fb9445", + "windows-amd64": "c8db13ebeda31497f354c0e950809db0ae9b2a2a69b8afee68c128c37300c157", +} DEFAULT_ENDPOINT = "https://storage.devino.ca" DEFAULT_BUCKET = "gh-artifacts" # MinIO maps the token's repository_owner_id claim to a policy of the same name @@ -246,24 +264,53 @@ def ensure_mc(endpoint): if dest.is_file(): return str(dest) dest_dir.mkdir(parents=True, exist_ok=True) + want = MC_SHA256.get(key) + if not want: + fail("No pinned sha256 for mc %s on %s; refusing to install an unverified binary" % (MC_VERSION, key)) + # Sources in order of preference: the org mirror, then the two public + # copies. dl.min.io is gone -- it has answered 410 Gone for every mc release + # since 2026-09-11/12 ("the MinIO Client project is archived ... these files + # are no longer served from this site"), which is what took every consumer + # job in the org down. The release assets of the archived github.com/minio/mc + # repository are the last public copy of this build, so they go last as a + # fallback rather than as something to depend on. + # TODO: populate the org mirror at + # storage.devino.ca/tools/mc/// (owner action -- + # the objects are not there today, the first URL 404s) so the primary source + # is org-controlled again and neither public source is on the critical path. urls = [ "%s/tools/mc/%s/%s/%s" % (endpoint.rstrip("/"), MC_VERSION, key, binname), "https://dl.min.io/client/mc/release/%s/archive/mc.%s" % (key, MC_VERSION), + "https://github.com/minio/mc/releases/download/%s/mc.%s.%s%s" + % (MC_VERSION, key, MC_VERSION, ".exe" if osn == "Windows" else ""), ] tmp = dest_dir / ("%s.%d.tmp" % (binname, os.getpid())) - last = None + errors = [] for url in urls: try: data = http(urllib.request.Request(url), timeout=180) + got = hashlib.sha256(data).hexdigest() + if got != want: + # Never install it, and do not stop: the next source may be + # intact. A wrong binary is a worse outcome than no binary. + errors.append( + (url, "sha256 mismatch: expected %s, got %s (%d bytes)" % (want, got, len(data))) + ) + continue with open(str(tmp), "wb") as f: f.write(data) os.chmod(str(tmp), 0o755) os.replace(str(tmp), str(dest)) - log("Installed mc %s from %s" % (MC_VERSION, url)) + log("Installed mc %s from %s (sha256 %s verified)" % (MC_VERSION, url, got)) return str(dest) except Exception as e: # noqa: BLE001 - last = e - fail("Could not download the MinIO client: %s" % last) + errors.append((url, str(e) or repr(e))) + # Name every source with its own error. When only the last one was reported, + # a 410 from dl.min.io read as if the org mirror had never been consulted. + fail( + "Could not download the MinIO client %s for %s; all %d source(s) failed:\n%s" + % (MC_VERSION, key, len(urls), "\n".join(" %s: %s" % (u, m) for u, m in errors)) + ) class Store(object): diff --git a/tests/test_ensure_mc.py b/tests/test_ensure_mc.py new file mode 100644 index 0000000..27a5d2f --- /dev/null +++ b/tests/test_ensure_mc.py @@ -0,0 +1,187 @@ +"""Unit tests for the `mc` download, its source list and its checksum pin. + +Standard library only, like lib/main.py itself: `python -m unittest discover -s tests`. +Every test patches `main.http`, so nothing here touches the network, dl.min.io, +github.com or the storage endpoint. + +The source list grew a third entry because dl.min.io began answering 410 Gone on +2026-09-11/12 (MinIO archived the client), which reds every consumer job: + + ##[error]Could not download the MinIO client: HTTP 410: 410 Gone +""" +import contextlib +import hashlib +import importlib.util +import io +import os +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_spec = importlib.util.spec_from_file_location( + "devino_artifact_main_mc", os.path.join(ROOT, "lib", "main.py") +) +main = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(main) + +ENDPOINT = "https://storage.devino.ca" +GONE = "HTTP 410: 410 Gone\nThe open-source MinIO Client (mc) project is archived." +GOOD = b"\x7fELF fake mc binary" +GOOD_SHA = hashlib.sha256(GOOD).hexdigest() +EVIL = b"\x7fELF something else entirely" + + +class FakeHTTP(object): + """Replays one reply per URL. A str reply is raised as RuntimeError.""" + + def __init__(self, replies): + self.replies = replies + self.urls = [] + + def __call__(self, req, timeout=None, **kwargs): + url = req.full_url if hasattr(req, "full_url") else req + self.urls.append(url) + reply = self.replies[min(len(self.urls) - 1, len(self.replies) - 1)] + if isinstance(reply, str): + raise RuntimeError(reply) + return reply + + +@contextlib.contextmanager +def runner(os_name="Linux", arch="X64", key="linux-amd64", replies=(GOOD,)): + """Run ensure_mc on a throwaway tool cache with `http` patched out.""" + cache = tempfile.mkdtemp(prefix="devino-mc-test-") + fake = FakeHTTP(list(replies)) + env = {"RUNNER_OS": os_name, "RUNNER_ARCH": arch, "RUNNER_TOOL_CACHE": cache} + buf = io.StringIO() + with mock.patch.dict(os.environ, env, clear=True), mock.patch.object( + main, "http", fake + ), mock.patch.dict(main.MC_SHA256, {key: GOOD_SHA}), contextlib.redirect_stdout(buf): + yield fake, cache, buf + + +def installed(cache, key, binname="mc"): + return os.path.join(cache, "devino-mc", main.MC_VERSION, key, binname) + + +class EnsureMcSources(unittest.TestCase): + def test_the_pinned_version_has_a_digest_for_every_supported_platform(self): + platforms = {"linux-amd64", "linux-arm64", "darwin-amd64", "darwin-arm64", "windows-amd64"} + self.assertEqual(set(main.MC_SHA256), platforms) + for key, digest in main.MC_SHA256.items(): + self.assertRegex(digest, r"^[0-9a-f]{64}$", key) + self.assertEqual(len(set(main.MC_SHA256.values())), len(platforms), "digests must differ") + + def test_mirror_is_tried_first_and_a_match_is_installed(self): + with runner() as (fake, cache, buf): + path = main.ensure_mc(ENDPOINT) + self.assertEqual(len(fake.urls), 1) + self.assertTrue(fake.urls[0].startswith(ENDPOINT + "/tools/mc/")) + self.assertEqual(path, installed(cache, "linux-amd64")) + with open(path, "rb") as f: + self.assertEqual(f.read(), GOOD) + self.assertIn("verified", buf.getvalue()) + + def test_falls_back_to_the_github_release_asset_when_both_mirrors_fail(self): + replies = ["HTTP 404: NoSuchKey", GONE, GOOD] + with runner(replies=replies) as (fake, cache, buf): + path = main.ensure_mc(ENDPOINT) + self.assertEqual(len(fake.urls), 3) + self.assertEqual( + fake.urls[2], + "https://github.com/minio/mc/releases/download/%s/mc.linux-amd64.%s" + % (main.MC_VERSION, main.MC_VERSION), + ) + self.assertTrue(os.path.isfile(path)) + + def test_windows_asks_for_the_exe_asset(self): + replies = ["HTTP 404: NoSuchKey", GONE, GOOD] + with runner("Windows", "X64", "windows-amd64", replies) as (fake, cache, buf): + path = main.ensure_mc(ENDPOINT) + self.assertEqual( + fake.urls[2], + "https://github.com/minio/mc/releases/download/%s/mc.windows-amd64.%s.exe" + % (main.MC_VERSION, main.MC_VERSION), + ) + self.assertEqual(path, installed(cache, "windows-amd64", "mc.exe")) + + def test_a_cached_binary_is_reused_without_any_download(self): + with runner() as (fake, cache, buf): + first = main.ensure_mc(ENDPOINT) + second = main.ensure_mc(ENDPOINT) + self.assertEqual(first, second) + self.assertEqual(len(fake.urls), 1, "second call must not hit the network") + + +class EnsureMcChecksum(unittest.TestCase): + def test_a_mismatched_payload_is_not_installed_and_the_next_source_is_tried(self): + with runner(replies=[EVIL, GONE, GOOD]) as (fake, cache, buf): + path = main.ensure_mc(ENDPOINT) + self.assertEqual(len(fake.urls), 3) + with open(path, "rb") as f: + self.assertEqual(f.read(), GOOD, "the tampered payload must never reach disk") + + def test_every_source_mismatching_fails_and_installs_nothing(self): + buf = io.StringIO() + with runner(replies=[EVIL]) as (fake, cache, buf): + with self.assertRaises(SystemExit): + main.ensure_mc(ENDPOINT) + self.assertEqual(len(fake.urls), 3) + self.assertFalse(os.path.exists(installed(cache, "linux-amd64"))) + text = buf.getvalue() + self.assertIn("sha256 mismatch", text) + self.assertIn("expected " + GOOD_SHA, text) + + def test_no_leftover_temp_file_after_a_failure(self): + with runner(replies=[EVIL]) as (fake, cache, buf): + with self.assertRaises(SystemExit): + main.ensure_mc(ENDPOINT) + d = os.path.dirname(installed(cache, "linux-amd64")) + self.assertEqual([f for f in os.listdir(d) if f.endswith(".tmp")], []) + + +class EnsureMcErrorReport(unittest.TestCase): + def _fail_text(self, replies): + with runner(replies=replies) as (fake, cache, buf): + with self.assertRaises(SystemExit): + main.ensure_mc(ENDPOINT) + return fake, buf.getvalue() + + def test_the_error_names_every_url_tried_one_per_line(self): + fake, text = self._fail_text(["HTTP 404: NoSuchKey", GONE, "HTTP 500: nope"]) + lines = text.splitlines() + for url in fake.urls: + self.assertEqual( + len([ln for ln in lines if ln.strip().startswith(url + ":")]), + 1, + "expected exactly one line for %s in:\n%s" % (url, text), + ) + + def test_each_url_keeps_its_own_error_not_only_the_last(self): + fake, text = self._fail_text(["HTTP 404: NoSuchKey", GONE, "HTTP 500: nope"]) + mirror, dlmin, github = fake.urls + self.assertRegex(text, re_line(mirror, "NoSuchKey")) + self.assertRegex(text, re_line(dlmin, "410 Gone")) + self.assertRegex(text, re_line(github, "HTTP 500")) + + def test_the_failure_is_a_single_error_annotation(self): + _, text = self._fail_text([GONE]) + self.assertEqual(len([ln for ln in text.splitlines() if ln.startswith("::error::")]), 1) + + def test_an_unsupported_platform_still_fails_before_any_download(self): + with runner("Plan9", "X64") as (fake, cache, buf): + with self.assertRaises(SystemExit): + main.ensure_mc(ENDPOINT) + self.assertEqual(fake.urls, []) + + +def re_line(url, needle): + import re + + return re.compile(r"%s:.*%s" % (re.escape(url), re.escape(needle))) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From f957040d57b906a91192948c4894b1d7110b0fe3 Mon Sep 17 00:00:00 2001 From: AminDhouib Date: Sat, 12 Sep 2026 04:04:53 -0400 Subject: [PATCH 2/3] docs(mc): correct the diagnosis -- the org mirror is populated, storage.devino.ca is misrouted The first commit said the mirror was never populated because it 404s. That is wrong: this repo's own CI installed mc from storage.devino.ca/tools/mc/// at 2026-09-12T00:37Z (run 34662229884), and uploaded 130 MiB through the same host in that run. storage.devino.ca now answers 404 on every path including /minio/health/live, and the response carries a Next.js content-security-policy, so the hostname is routed to the wrong backend rather than MinIO being empty or down. That also explains the "MinIO STS AssumeRoleWithWebIdentity failed: HTTP 404" this branch hits in CI after mc installs successfully. The code change is unaffected; only the comment, the README and the follow-up are re-pointed at restoring the route. --- README.md | 8 ++++---- lib/main.py | 27 +++++++++++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3003dee..890fa83 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,10 @@ to GitHub's blob store. This action writes to devino instead. If every source fails, the error names each URL with its own reason. `dl.min.io` has answered `410 Gone` since 2026-09-11/12 — MinIO archived the - client and stopped serving those files — so today the GitHub release assets - are the only source that responds, and the mirror should be populated at - `storage.devino.ca/tools/mc///mc`. Re-pin `MC_SHA256` - from each release's `.sha256sum` asset whenever `MC_VERSION` changes. + client and stopped serving those files. The org mirror is populated and is + normally the source that answers, so the third entry exists for the case + where it is not reachable. Re-pin `MC_SHA256` from each release's + `.sha256sum` asset whenever `MC_VERSION` changes. 3. Upload: matched files are packed into one `.tgz` whose root mirrors upstream semantics (a single directory uploads its contents; several paths share their least common ancestor), then copied to diff --git a/lib/main.py b/lib/main.py index 8c33639..af71f0e 100644 --- a/lib/main.py +++ b/lib/main.py @@ -268,16 +268,23 @@ def ensure_mc(endpoint): if not want: fail("No pinned sha256 for mc %s on %s; refusing to install an unverified binary" % (MC_VERSION, key)) # Sources in order of preference: the org mirror, then the two public - # copies. dl.min.io is gone -- it has answered 410 Gone for every mc release - # since 2026-09-11/12 ("the MinIO Client project is archived ... these files - # are no longer served from this site"), which is what took every consumer - # job in the org down. The release assets of the archived github.com/minio/mc - # repository are the last public copy of this build, so they go last as a - # fallback rather than as something to depend on. - # TODO: populate the org mirror at - # storage.devino.ca/tools/mc/// (owner action -- - # the objects are not there today, the first URL 404s) so the primary source - # is org-controlled again and neither public source is on the critical path. + # copies. The mirror is populated and normally serves this build -- it did + # as recently as 2026-09-12T00:37Z (this repo's own CI, run 34662229884) -- + # but storage.devino.ca is misrouted as of 2026-09-12: every path, including + # /minio/health/live, answers 404 from some other application, so the mirror + # is unreachable rather than empty. + # + # dl.min.io is gone for good: 410 Gone for every mc release since + # 2026-09-11/12 ("the MinIO Client project is archived ... these files are + # no longer served from this site"). The mirror being down and the secondary + # being retired on the same day left no source at all, which is what took + # every consumer job in the org down here. The release assets of the + # archived github.com/minio/mc repository are the last public copy of this + # build, so they go last: a fallback, not something to depend on. + # + # TODO: restore the storage.devino.ca route (owner action). This third URL + # only un-breaks the download step; STS and every `mc cp`/`mc ls` in this + # file still go to that host, so the action cannot work until it is back. urls = [ "%s/tools/mc/%s/%s/%s" % (endpoint.rstrip("/"), MC_VERSION, key, binname), "https://dl.min.io/client/mc/release/%s/archive/mc.%s" % (key, MC_VERSION), From ea828c5754195127fd59be297ea145e77d6ce84b Mon Sep 17 00:00:00 2001 From: AminDhouib Date: Sat, 12 Sep 2026 04:10:28 -0400 Subject: [PATCH 3/3] docs(mc): the mirror was down, not misrouted -- shared MinIO had no restart policy Correcting the second commit. The 404 from storage.devino.ca was a symptom, not the cause. The Docker daemon on the storage host restarted around 05:00Z on 2026-09-12 and the shared MinIO compose has no `restart:` policy, so its container stayed exited (255) while every other app on the host came back. With no container there was no Traefik router for the hostname, so requests fell through to another app and 404d every MinIO path. The same failure appears twice in that service's deploy history as "Redeploy shared MinIO - was returning 404". Starting the container restored it at 08:10Z. The TODO is now the actual fix: add `restart: unless-stopped` to the shared-minio compose. Re-verified with the mirror back up -- all five mirrored binaries are byte-identical to the pinned digests, so the checksum gate does not disturb the primary source. --- README.md | 3 ++- lib/main.py | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 890fa83..c8f2b71 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,8 @@ to GitHub's blob store. This action writes to devino instead. `dl.min.io` has answered `410 Gone` since 2026-09-11/12 — MinIO archived the client and stopped serving those files. The org mirror is populated and is normally the source that answers, so the third entry exists for the case - where it is not reachable. Re-pin `MC_SHA256` from each release's + where the mirror itself is down: with the secondary retired there is no + longer anything behind it. Re-pin `MC_SHA256` from each release's `.sha256sum` asset whenever `MC_VERSION` changes. 3. Upload: matched files are packed into one `.tgz` whose root mirrors upstream semantics (a single directory uploads its contents; several diff --git a/lib/main.py b/lib/main.py index af71f0e..bf66364 100644 --- a/lib/main.py +++ b/lib/main.py @@ -268,11 +268,17 @@ def ensure_mc(endpoint): if not want: fail("No pinned sha256 for mc %s on %s; refusing to install an unverified binary" % (MC_VERSION, key)) # Sources in order of preference: the org mirror, then the two public - # copies. The mirror is populated and normally serves this build -- it did - # as recently as 2026-09-12T00:37Z (this repo's own CI, run 34662229884) -- - # but storage.devino.ca is misrouted as of 2026-09-12: every path, including - # /minio/health/live, answers 404 from some other application, so the mirror - # is unreachable rather than empty. + # copies. The mirror is populated and is normally the source that answers. + # + # On 2026-09-12 it did not. The Docker daemon on the storage host restarted + # around 05:00Z; the shared MinIO compose has no `restart:` policy, so its + # container stayed exited (255) while every other app on the host came back. + # With no container, Traefik had no router for storage.devino.ca and the + # requests fell through to another app, which 404s every MinIO path -- + # including /minio/health/live -- and answers with that app's headers. The + # same failure appears twice in this service's deploy history as "Redeploy + # shared MinIO - was returning 404". Starting the container restored it at + # 08:10Z. So the mirror was down, not empty, and the 404 was a symptom. # # dl.min.io is gone for good: 410 Gone for every mc release since # 2026-09-11/12 ("the MinIO Client project is archived ... these files are @@ -282,9 +288,9 @@ def ensure_mc(endpoint): # archived github.com/minio/mc repository are the last public copy of this # build, so they go last: a fallback, not something to depend on. # - # TODO: restore the storage.devino.ca route (owner action). This third URL - # only un-breaks the download step; STS and every `mc cp`/`mc ls` in this - # file still go to that host, so the action cannot work until it is back. + # TODO: add `restart: unless-stopped` to the shared-minio compose (owner + # action) so a daemon restart cannot take the mirror -- and with it STS and + # every `mc cp`/`mc ls` in this file -- down until someone notices. urls = [ "%s/tools/mc/%s/%s/%s" % (endpoint.rstrip("/"), MC_VERSION, key, binname), "https://dl.min.io/client/mc/release/%s/archive/mc.%s" % (key, MC_VERSION),