ci: add nightly job to sync source mirror with pkgs in builtin

.github/workflows/bin/find-missing-mirror-specs.py

@@ -0,0 +1,75 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Print one ``pkg@=version`` spec per line for every checksummed package
+version that is not yet present in the source mirror.
+
+Usage: spack python find-missing-mirror-specs.py <sha256-file>
+
+``<sha256-file>`` contains one sha256 digest per line, obtained by listing the
+content-addressed ``_source-cache/archive/`` prefix of the mirror. Only
+URL-based versions with a ``sha256`` are considered, which matches the
+content-addressed layout (``archive/<sha256[:2]>/<sha256>.<ext>``) used by
+``spack mirror create`` and mirrors the filtering done by
+``spack repo show-version-updates --no-manual-packages --only-redistributable
+--no-git-versions``.
+"""
+
+import sys
+from typing import Dict, List
+
+import spack.repo
+import spack.spec
+from spack.util import tty
+from spack.version import StandardVersion
+
+
+def main(sha256_file: str) -> None:
+    with open(sha256_file) as f:
+        # Store shas as a set / hash-table for faster key lookups
+        mirrored = {line.strip() for line in f if line.strip()}
+
+    specs_to_output: List[spack.spec.Spec] = []
+
+    repo = spack.repo.PATH.get_repo("builtin")
+
+    for pkg_cls in repo.all_package_classes():
+        # Filter out manual packages
+        if pkg_cls.manual_download:
+            continue
+
+        # Get all versions with checksums; no sha256 means not a
+        # content-addressed URL fetch (e.g. a git version)
+        version_to_checksum: Dict[StandardVersion, str] = {
+            version: version_dict["sha256"]
+            for version, version_dict in pkg_cls.versions.items()
+            if "sha256" in version_dict
+        }
+
+        for version, sha256 in version_to_checksum.items():
+            if sha256 in mirrored:
+                continue
+            version_spec = spack.spec.Spec(pkg_cls.name)
+            version_spec.constrain(f"@={version}")
+            specs_to_output.append(version_spec)
+
+    # Filter out non-redistributable packages
+    specs_to_output = [
+        spec for spec in specs_to_output if repo.get_pkg_class(spec.name).redistribute_source(spec)
+    ]
+
+    # Output specs one per line for use by `spack mirror create`
+    # limit to a maximum of 100 specs at a time due to GitHub
+    # runner disk space limitations. Skipped specs will be
+    # retried on the next scheduled job.
+    specs_to_output_num = len(specs_to_output)
+    if specs_to_output_num > 100:
+        tty.warn(f"Limiting to first 100 missing specs. Detected {specs_to_output_num} missing.")
+
+    for spec in specs_to_output[:100]:
+        print(spec)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1])

.github/workflows/sync-src-mirror.yml

@@ -0,0 +1,92 @@
+name: sync-src-mirror
+
+# Nightly backfill of the source mirror: finds package versions missing from
+# the mirror (e.g. because update-src-mirror failed or a fetch was flaky) and
+# mirrors them.
+
+on:
+  schedule:
+    - cron: '0 3 * * *'  # nightly at 03:00 UTC
+  workflow_dispatch:
+
+permissions:
+  id-token: write  # Required for AWS OIDC authentication
+  contents: read
+
+jobs:
+  sync-mirror:
+    if: github.repository == 'spack/spack-packages'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+
+      - name: Checkout Spack
+        uses: ./.github/actions/checkout-spack
+        with:
+          fetch-depth: 1
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c  # v6.2.3
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Get digests already in the mirror
+        run: |
+          # Tarballs are content addressed as _source-cache/archive/<xx>/<sha256>.<ext>.
+          # Thus a single recursive listing of the archive/ prefix is a complete list
+          # of mirrored versions, patches, and package resources. Use awk and sed to strip
+          # out the shas from the filenames and save the shas to a file for later use.
+          aws s3 ls --recursive "s3://${{ secrets.S3_BUCKET_NAME }}/_source-cache/archive/" \
+            | awk '{print $4}' \
+            | sed -e 's|.*/||' -e 's|\..*||' \
+            | sort -u > mirrored-shas.txt
+          echo "$(wc -l < mirrored-shas.txt) shas in the mirror"
+
+      - name: Find shas in repository missing from mirror
+        id: find-missing
+        run: |
+          source spack-core/share/spack/setup-env.sh
+
+          spack python .github/workflows/bin/find-missing-mirror-specs.py \
+            mirrored-shas.txt > missing-specs.txt
+
+          if [ ! -s missing-specs.txt ]; then
+            echo "Mirror is up to date"
+            echo "missing=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          echo "$(wc -l < missing-specs.txt) versions missing from the mirror:"
+          cat missing-specs.txt
+          echo "missing=true" >> $GITHUB_OUTPUT
+
+      - name: Create source mirror
+        id: create-mirror
+        if: steps.find-missing.outputs.missing == 'true'
+        run: |
+          source spack-core/share/spack/setup-env.sh
+
+          # Tolerate individual fetch failures (dead upstream URLs etc.) and
+          # upload whatever was mirrored successfully; anything that failed
+          # will simply be retried on the next nightly run.
+          spack -c concretizer:unify:false -c config:deprecated:true \
+            mirror create -d src-mirror --jobs 8 --file missing-specs.txt \
+            || echo "Some sources failed to fetch; uploading the rest"
+
+          if [ -d src-mirror/_source-cache ]; then
+            echo "upload=true" >> $GITHUB_OUTPUT
+          else
+            echo "Nothing was mirrored"
+            echo "upload=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Upload to S3
+        if: steps.create-mirror.outputs.upload == 'true'
+        run: |
+          aws s3 cp src-mirror/_source-cache s3://${{ secrets.S3_BUCKET_NAME }}/_source-cache/ \
+            --no-overwrite \
+            --recursive \
+            --no-progress
+          echo "Successfully uploaded source mirror to S3"