fix(mc): fall back to the archived minio/mc GitHub release assets and verify every download against a pinned sha256 (dl.min.io answers 410)

README.md

@@ -35,8 +35,20 @@ to GitHub's blob store. This action writes to devino instead.
    MinIO STS (`AssumeRoleWithWebIdentity`) for one-hour credentials. MinIO picks the policy named after the token's
    `repository_owner_id` claim, so only workflows owned by DevinoSolutions get
    access; tokens from any other owner map to no policy and are refused.
-2. A pinned `mc` (MinIO client) is fetched from `storage.devino.ca/tools/`
-   (fallback: dl.min.io) and cached in the runner tool cache.
+2. A pinned `mc` (MinIO client) is fetched from `storage.devino.ca/tools/` and
+   cached in the runner tool cache. Two fallbacks follow the org mirror:
+   `dl.min.io`, and the release assets of the archived `github.com/minio/mc`
+   repository. Whichever source answers, the bytes are checked against the
+   per-platform sha256 pinned in `lib/main.py` (`MC_SHA256`) before the binary
+   is installed; a mismatch is never installed and the next source is tried.
+   If every source fails, the error names each URL with its own reason.
+
+   `dl.min.io` has answered `410 Gone` since 2026-09-11/12 — MinIO archived the
+   client and stopped serving those files. The org mirror is populated and is
+   normally the source that answers, so the third entry exists for the case
+   where the mirror itself is down: with the secondary retired there is no
+   longer anything behind it. Re-pin `MC_SHA256` from each release's
+   `.sha256sum` asset whenever `MC_VERSION` changes.
 3. Upload: matched files are packed into one `.tgz` whose root mirrors
    upstream semantics (a single directory uploads its contents; several
    paths share their least common ancestor), then copied to

lib/main.py

@@ -30,6 +30,24 @@
 import urllib.request
 
 MC_VERSION = "RELEASE.2025-08-13T08-35-41Z"
+# sha256 of the pinned `mc` binary for each supported platform. Every download,
+# from whichever source, is checked against this table before it is installed:
+# the primary source is an org-controlled bucket and the last resort is an
+# archived third-party repository, so the pin is what makes the three sources
+# interchangeable instead of three different trust levels.
+#
+# Taken on 2026-09-12 from the `.sha256sum` asset published next to each binary
+# on the archived upstream release
+# https://github.com/minio/mc/releases/tag/RELEASE.2025-08-13T08-35-41Z
+# (e.g. mc.linux-amd64.RELEASE.2025-08-13T08-35-41Z.sha256sum). Re-pin these
+# whenever MC_VERSION changes.
+MC_SHA256 = {
+    "linux-amd64": "01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891",
+    "linux-arm64": "14c8c9616cfce4636add161304353244e8de383b2e2752c0e9dad01d4c27c12c",
+    "darwin-amd64": "2862c79cce11b09be9a8911a279b2e9465bebf74b9f01abca9c348a0d795f0cb",
+    "darwin-arm64": "a877fd0c183409da9f20f9d6e1811987298bbbca1aa03428eebdffba79fb9445",
+    "windows-amd64": "c8db13ebeda31497f354c0e950809db0ae9b2a2a69b8afee68c128c37300c157",
+}
 DEFAULT_ENDPOINT = "https://storage.devino.ca"
 DEFAULT_BUCKET = "gh-artifacts"
 # MinIO maps the token's repository_owner_id claim to a policy of the same name
@@ -246,24 +264,66 @@ def ensure_mc(endpoint):
     if dest.is_file():
         return str(dest)
     dest_dir.mkdir(parents=True, exist_ok=True)
+    want = MC_SHA256.get(key)
+    if not want:
+        fail("No pinned sha256 for mc %s on %s; refusing to install an unverified binary" % (MC_VERSION, key))
+    # Sources in order of preference: the org mirror, then the two public
+    # copies. The mirror is populated and is normally the source that answers.
+    #
+    # On 2026-09-12 it did not. The Docker daemon on the storage host restarted
+    # around 05:00Z; the shared MinIO compose has no `restart:` policy, so its
+    # container stayed exited (255) while every other app on the host came back.
+    # With no container, Traefik had no router for storage.devino.ca and the
+    # requests fell through to another app, which 404s every MinIO path --
+    # including /minio/health/live -- and answers with that app's headers. The
+    # same failure appears twice in this service's deploy history as "Redeploy
+    # shared MinIO - was returning 404". Starting the container restored it at
+    # 08:10Z. So the mirror was down, not empty, and the 404 was a symptom.
+    #
+    # dl.min.io is gone for good: 410 Gone for every mc release since
+    # 2026-09-11/12 ("the MinIO Client project is archived ... these files are
+    # no longer served from this site"). The mirror being down and the secondary
+    # being retired on the same day left no source at all, which is what took
+    # every consumer job in the org down here. The release assets of the
+    # archived github.com/minio/mc repository are the last public copy of this
+    # build, so they go last: a fallback, not something to depend on.
+    #
+    # TODO: add `restart: unless-stopped` to the shared-minio compose (owner
+    # action) so a daemon restart cannot take the mirror -- and with it STS and
+    # every `mc cp`/`mc ls` in this file -- down until someone notices.
     urls = [
         "%s/tools/mc/%s/%s/%s" % (endpoint.rstrip("/"), MC_VERSION, key, binname),
         "https://dl.min.io/client/mc/release/%s/archive/mc.%s" % (key, MC_VERSION),
+        "https://github.com/minio/mc/releases/download/%s/mc.%s.%s%s"
+        % (MC_VERSION, key, MC_VERSION, ".exe" if osn == "Windows" else ""),
     ]
     tmp = dest_dir / ("%s.%d.tmp" % (binname, os.getpid()))
-    last = None
+    errors = []
     for url in urls:
         try:
             data = http(urllib.request.Request(url), timeout=180)
+            got = hashlib.sha256(data).hexdigest()
+            if got != want:
+                # Never install it, and do not stop: the next source may be
+                # intact. A wrong binary is a worse outcome than no binary.
+                errors.append(
+                    (url, "sha256 mismatch: expected %s, got %s (%d bytes)" % (want, got, len(data)))
+                )
+                continue
             with open(str(tmp), "wb") as f:
                 f.write(data)
             os.chmod(str(tmp), 0o755)
             os.replace(str(tmp), str(dest))
-            log("Installed mc %s from %s" % (MC_VERSION, url))
+            log("Installed mc %s from %s (sha256 %s verified)" % (MC_VERSION, url, got))
             return str(dest)
         except Exception as e:  # noqa: BLE001
-            last = e
-    fail("Could not download the MinIO client: %s" % last)
+            errors.append((url, str(e) or repr(e)))
+    # Name every source with its own error. When only the last one was reported,
+    # a 410 from dl.min.io read as if the org mirror had never been consulted.
+    fail(
+        "Could not download the MinIO client %s for %s; all %d source(s) failed:\n%s"
+        % (MC_VERSION, key, len(urls), "\n".join("  %s: %s" % (u, m) for u, m in errors))
+    )
 
 
 class Store(object):

tests/test_ensure_mc.py

@@ -0,0 +1,187 @@
+"""Unit tests for the `mc` download, its source list and its checksum pin.
+
+Standard library only, like lib/main.py itself: `python -m unittest discover -s tests`.
+Every test patches `main.http`, so nothing here touches the network, dl.min.io,
+github.com or the storage endpoint.
+
+The source list grew a third entry because dl.min.io began answering 410 Gone on
+2026-09-11/12 (MinIO archived the client), which reds every consumer job:
+
+    ##[error]Could not download the MinIO client: HTTP 410: 410 Gone
+"""
+import contextlib
+import hashlib
+import importlib.util
+import io
+import os
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_spec = importlib.util.spec_from_file_location(
+    "devino_artifact_main_mc", os.path.join(ROOT, "lib", "main.py")
+)
+main = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(main)
+
+ENDPOINT = "https://storage.devino.ca"
+GONE = "HTTP 410: 410 Gone\nThe open-source MinIO Client (mc) project is archived."
+GOOD = b"\x7fELF fake mc binary"
+GOOD_SHA = hashlib.sha256(GOOD).hexdigest()
+EVIL = b"\x7fELF something else entirely"
+
+
+class FakeHTTP(object):
+    """Replays one reply per URL. A str reply is raised as RuntimeError."""
+
+    def __init__(self, replies):
+        self.replies = replies
+        self.urls = []
+
+    def __call__(self, req, timeout=None, **kwargs):
+        url = req.full_url if hasattr(req, "full_url") else req
+        self.urls.append(url)
+        reply = self.replies[min(len(self.urls) - 1, len(self.replies) - 1)]
+        if isinstance(reply, str):
+            raise RuntimeError(reply)
+        return reply
+
+
+@contextlib.contextmanager
+def runner(os_name="Linux", arch="X64", key="linux-amd64", replies=(GOOD,)):
+    """Run ensure_mc on a throwaway tool cache with `http` patched out."""
+    cache = tempfile.mkdtemp(prefix="devino-mc-test-")
+    fake = FakeHTTP(list(replies))
+    env = {"RUNNER_OS": os_name, "RUNNER_ARCH": arch, "RUNNER_TOOL_CACHE": cache}
+    buf = io.StringIO()
+    with mock.patch.dict(os.environ, env, clear=True), mock.patch.object(
+        main, "http", fake
+    ), mock.patch.dict(main.MC_SHA256, {key: GOOD_SHA}), contextlib.redirect_stdout(buf):
+        yield fake, cache, buf
+
+
+def installed(cache, key, binname="mc"):
+    return os.path.join(cache, "devino-mc", main.MC_VERSION, key, binname)
+
+
+class EnsureMcSources(unittest.TestCase):
+    def test_the_pinned_version_has_a_digest_for_every_supported_platform(self):
+        platforms = {"linux-amd64", "linux-arm64", "darwin-amd64", "darwin-arm64", "windows-amd64"}
+        self.assertEqual(set(main.MC_SHA256), platforms)
+        for key, digest in main.MC_SHA256.items():
+            self.assertRegex(digest, r"^[0-9a-f]{64}$", key)
+        self.assertEqual(len(set(main.MC_SHA256.values())), len(platforms), "digests must differ")
+
+    def test_mirror_is_tried_first_and_a_match_is_installed(self):
+        with runner() as (fake, cache, buf):
+            path = main.ensure_mc(ENDPOINT)
+        self.assertEqual(len(fake.urls), 1)
+        self.assertTrue(fake.urls[0].startswith(ENDPOINT + "/tools/mc/"))
+        self.assertEqual(path, installed(cache, "linux-amd64"))
+        with open(path, "rb") as f:
+            self.assertEqual(f.read(), GOOD)
+        self.assertIn("verified", buf.getvalue())
+
+    def test_falls_back_to_the_github_release_asset_when_both_mirrors_fail(self):
+        replies = ["HTTP 404: NoSuchKey", GONE, GOOD]
+        with runner(replies=replies) as (fake, cache, buf):
+            path = main.ensure_mc(ENDPOINT)
+        self.assertEqual(len(fake.urls), 3)
+        self.assertEqual(
+            fake.urls[2],
+            "https://github.com/minio/mc/releases/download/%s/mc.linux-amd64.%s"
+            % (main.MC_VERSION, main.MC_VERSION),
+        )
+        self.assertTrue(os.path.isfile(path))
+
+    def test_windows_asks_for_the_exe_asset(self):
+        replies = ["HTTP 404: NoSuchKey", GONE, GOOD]
+        with runner("Windows", "X64", "windows-amd64", replies) as (fake, cache, buf):
+            path = main.ensure_mc(ENDPOINT)
+        self.assertEqual(
+            fake.urls[2],
+            "https://github.com/minio/mc/releases/download/%s/mc.windows-amd64.%s.exe"
+            % (main.MC_VERSION, main.MC_VERSION),
+        )
+        self.assertEqual(path, installed(cache, "windows-amd64", "mc.exe"))
+
+    def test_a_cached_binary_is_reused_without_any_download(self):
+        with runner() as (fake, cache, buf):
+            first = main.ensure_mc(ENDPOINT)
+            second = main.ensure_mc(ENDPOINT)
+        self.assertEqual(first, second)
+        self.assertEqual(len(fake.urls), 1, "second call must not hit the network")
+
+
+class EnsureMcChecksum(unittest.TestCase):
+    def test_a_mismatched_payload_is_not_installed_and_the_next_source_is_tried(self):
+        with runner(replies=[EVIL, GONE, GOOD]) as (fake, cache, buf):
+            path = main.ensure_mc(ENDPOINT)
+        self.assertEqual(len(fake.urls), 3)
+        with open(path, "rb") as f:
+            self.assertEqual(f.read(), GOOD, "the tampered payload must never reach disk")
+
+    def test_every_source_mismatching_fails_and_installs_nothing(self):
+        buf = io.StringIO()
+        with runner(replies=[EVIL]) as (fake, cache, buf):
+            with self.assertRaises(SystemExit):
+                main.ensure_mc(ENDPOINT)
+        self.assertEqual(len(fake.urls), 3)
+        self.assertFalse(os.path.exists(installed(cache, "linux-amd64")))
+        text = buf.getvalue()
+        self.assertIn("sha256 mismatch", text)
+        self.assertIn("expected " + GOOD_SHA, text)
+
+    def test_no_leftover_temp_file_after_a_failure(self):
+        with runner(replies=[EVIL]) as (fake, cache, buf):
+            with self.assertRaises(SystemExit):
+                main.ensure_mc(ENDPOINT)
+        d = os.path.dirname(installed(cache, "linux-amd64"))
+        self.assertEqual([f for f in os.listdir(d) if f.endswith(".tmp")], [])
+
+
+class EnsureMcErrorReport(unittest.TestCase):
+    def _fail_text(self, replies):
+        with runner(replies=replies) as (fake, cache, buf):
+            with self.assertRaises(SystemExit):
+                main.ensure_mc(ENDPOINT)
+        return fake, buf.getvalue()
+
+    def test_the_error_names_every_url_tried_one_per_line(self):
+        fake, text = self._fail_text(["HTTP 404: NoSuchKey", GONE, "HTTP 500: nope"])
+        lines = text.splitlines()
+        for url in fake.urls:
+            self.assertEqual(
+                len([ln for ln in lines if ln.strip().startswith(url + ":")]),
+                1,
+                "expected exactly one line for %s in:\n%s" % (url, text),
+            )
+
+    def test_each_url_keeps_its_own_error_not_only_the_last(self):
+        fake, text = self._fail_text(["HTTP 404: NoSuchKey", GONE, "HTTP 500: nope"])
+        mirror, dlmin, github = fake.urls
+        self.assertRegex(text, re_line(mirror, "NoSuchKey"))
+        self.assertRegex(text, re_line(dlmin, "410 Gone"))
+        self.assertRegex(text, re_line(github, "HTTP 500"))
+
+    def test_the_failure_is_a_single_error_annotation(self):
+        _, text = self._fail_text([GONE])
+        self.assertEqual(len([ln for ln in text.splitlines() if ln.startswith("::error::")]), 1)
+
+    def test_an_unsupported_platform_still_fails_before_any_download(self):
+        with runner("Plan9", "X64") as (fake, cache, buf):
+            with self.assertRaises(SystemExit):
+                main.ensure_mc(ENDPOINT)
+        self.assertEqual(fake.urls, [])
+
+
+def re_line(url, needle):
+    import re
+
+    return re.compile(r"%s:.*%s" % (re.escape(url), re.escape(needle)))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)