From 3f88e2124eb1448e5f40080287845885d4d73896 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Wed, 12 Aug 2026 15:05:56 -0700 Subject: [PATCH 1/3] ci: add nightly job to sync source mirror with pkgs in builtin Signed-off-by: Alec Scott --- .../bin/find-missing-mirror-specs.py | 67 ++++++++++++++ .github/workflows/sync-src-mirror.yml | 92 +++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 .github/workflows/bin/find-missing-mirror-specs.py create mode 100644 .github/workflows/sync-src-mirror.yml diff --git a/.github/workflows/bin/find-missing-mirror-specs.py b/.github/workflows/bin/find-missing-mirror-specs.py new file mode 100644 index 00000000000..f7d0091b7e5 --- /dev/null +++ b/.github/workflows/bin/find-missing-mirror-specs.py @@ -0,0 +1,67 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Print one ``pkg@=version`` spec per line for every checksummed package +version that is not yet present in the source mirror. + +Usage: spack python find-missing-mirror-specs.py + +```` contains one sha256 digest per line, obtained by listing the +content-addressed ``_source-cache/archive/`` prefix of the mirror. Only +URL-based versions with a ``sha256`` are considered, which matches the +content-addressed layout (``archive//.``) used by +``spack mirror create`` and mirrors the filtering done by +``spack repo show-version-updates --no-manual-packages --only-redistributable +--no-git-versions``. +""" + +import sys +from typing import Dict, List + +import spack.repo +import spack.spec +from spack.version import StandardVersion + + +def main(sha256_file: str) -> None: + with open(sha256_file) as f: + # store shas as a set / hash-table for faster key lookups + mirrored = {line.strip() for line in f if line.strip()} + + specs_to_output: List[spack.spec.Spec] = [] + + repo = spack.repo.PATH.get_repo("builtin") + + for pkg_cls in repo.all_package_classes(): + # filter out manual packages + if pkg_cls.manual_download: + continue + + # get all versions with checksums; no sha256 means not a + # content-addressed URL fetch (e.g. a git version) + version_to_checksum: Dict[StandardVersion, str] = { + version: version_dict["sha256"] + for version, version_dict in pkg_cls.versions.items() + if "sha256" in version_dict + } + + for version, sha256 in version_to_checksum.items(): + if sha256 in mirrored: + continue + version_spec = spack.spec.Spec(pkg_cls.name) + version_spec.constrain(f"@={version}") + specs_to_output.append(version_spec) + + # filter out non-redistributable packages + specs_to_output = [ + spec for spec in specs_to_output if repo.get_pkg_class(spec.name).redistribute_source(spec) + ] + + # output specs one per line for use by `spack mirror create` + for spec in specs_to_output: + print(spec) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/.github/workflows/sync-src-mirror.yml b/.github/workflows/sync-src-mirror.yml new file mode 100644 index 00000000000..903d474d017 --- /dev/null +++ b/.github/workflows/sync-src-mirror.yml @@ -0,0 +1,92 @@ +name: sync-src-mirror + +# Nightly backfill of the source mirror: finds package versions missing from +# the mirror (e.g. because update-src-mirror failed or a fetch was flaky) and +# mirrors them. + +on: + schedule: + - cron: '0 3 * * *' # nightly at 03:00 UTC + workflow_dispatch: + +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read + +jobs: + sync-mirror: + if: github.repository == 'spack/spack-packages' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Checkout Spack + uses: ./.github/actions/checkout-spack + with: + fetch-depth: 1 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Get digests already in the mirror + run: | + # Tarballs are content addressed as _source-cache/archive//.. + # Thus a single recursive listing of the archive/ prefix is a complete list + # of mirrored versions, patches, and package resources. Use awk and sed to strip + # out the shas from the filenames and save the shas to a file for later use. + aws s3 ls --recursive "s3://${{ secrets.S3_BUCKET_NAME }}/_source-cache/archive/" \ + | awk '{print $4}' \ + | sed -e 's|.*/||' -e 's|\..*||' \ + | sort -u > mirrored-shas.txt + echo "$(wc -l < mirrored-shas.txt) shas in the mirror" + + - name: Find shas in repository missing from mirror + id: find-missing + run: | + source spack-core/share/spack/setup-env.sh + + spack python .github/workflows/bin/find-missing-mirror-specs.py \ + mirrored-shas.txt > missing-specs.txt + + if [ ! -s missing-specs.txt ]; then + echo "Mirror is up to date" + echo "missing=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "$(wc -l < missing-specs.txt) versions missing from the mirror:" + cat missing-specs.txt + echo "missing=true" >> $GITHUB_OUTPUT + + - name: Create source mirror + id: create-mirror + if: steps.find-missing.outputs.missing == 'true' + run: | + source spack-core/share/spack/setup-env.sh + + # Tolerate individual fetch failures (dead upstream URLs etc.) and + # upload whatever was mirrored successfully; anything that failed + # will simply be retried on the next nightly run. + spack -c concretizer:unify:false -c config:deprecated:true \ + mirror create -d src-mirror --jobs 8 --file missing-specs.txt \ + || echo "Some sources failed to fetch; uploading the rest" + + if [ -d src-mirror/_source-cache ]; then + echo "upload=true" >> $GITHUB_OUTPUT + else + echo "Nothing was mirrored" + echo "upload=false" >> $GITHUB_OUTPUT + fi + + - name: Upload to S3 + if: steps.create-mirror.outputs.upload == 'true' + run: | + aws s3 cp src-mirror/_source-cache s3://${{ secrets.S3_BUCKET_NAME }}/_source-cache/ \ + --no-overwrite \ + --recursive \ + --no-progress + echo "Successfully uploaded source mirror to S3" From 19bcc5ff7343f023aa7c71390a35fdb4fb34611f Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Wed, 26 Aug 2026 08:51:45 -0700 Subject: [PATCH 2/3] Limit job to first 100 missing specs Signed-off-by: Alec Scott --- .../workflows/bin/find-missing-mirror-specs.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bin/find-missing-mirror-specs.py b/.github/workflows/bin/find-missing-mirror-specs.py index f7d0091b7e5..f7d016adbe4 100644 --- a/.github/workflows/bin/find-missing-mirror-specs.py +++ b/.github/workflows/bin/find-missing-mirror-specs.py @@ -26,7 +26,7 @@ def main(sha256_file: str) -> None: with open(sha256_file) as f: - # store shas as a set / hash-table for faster key lookups + # Store shas as a set / hash-table for faster key lookups mirrored = {line.strip() for line in f if line.strip()} specs_to_output: List[spack.spec.Spec] = [] @@ -34,11 +34,11 @@ def main(sha256_file: str) -> None: repo = spack.repo.PATH.get_repo("builtin") for pkg_cls in repo.all_package_classes(): - # filter out manual packages + # Filter out manual packages if pkg_cls.manual_download: continue - # get all versions with checksums; no sha256 means not a + # Get all versions with checksums; no sha256 means not a # content-addressed URL fetch (e.g. a git version) version_to_checksum: Dict[StandardVersion, str] = { version: version_dict["sha256"] @@ -53,13 +53,16 @@ def main(sha256_file: str) -> None: version_spec.constrain(f"@={version}") specs_to_output.append(version_spec) - # filter out non-redistributable packages + # Filter out non-redistributable packages specs_to_output = [ spec for spec in specs_to_output if repo.get_pkg_class(spec.name).redistribute_source(spec) ] - # output specs one per line for use by `spack mirror create` - for spec in specs_to_output: + # Output specs one per line for use by `spack mirror create` + # limit to a maximum of 100 specs at a time due to GitHub + # runner disk space limitations. Skipped specs will be + # retried on the next scheduled job. + for spec in specs_to_output[:100]: print(spec) From 2c35981df779073d690838b56ca55f6148f4f16a Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Wed, 26 Aug 2026 09:13:01 -0700 Subject: [PATCH 3/3] Add warning when >100 missing specs are detected Signed-off-by: Alec Scott --- .github/workflows/bin/find-missing-mirror-specs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/bin/find-missing-mirror-specs.py b/.github/workflows/bin/find-missing-mirror-specs.py index f7d016adbe4..3ae83a4f4c3 100644 --- a/.github/workflows/bin/find-missing-mirror-specs.py +++ b/.github/workflows/bin/find-missing-mirror-specs.py @@ -21,6 +21,7 @@ import spack.repo import spack.spec +from spack.util import tty from spack.version import StandardVersion @@ -62,6 +63,10 @@ def main(sha256_file: str) -> None: # limit to a maximum of 100 specs at a time due to GitHub # runner disk space limitations. Skipped specs will be # retried on the next scheduled job. + specs_to_output_num = len(specs_to_output) + if specs_to_output_num > 100: + tty.warn(f"Limiting to first 100 missing specs. Detected {specs_to_output_num} missing.") + for spec in specs_to_output[:100]: print(spec)