diff --git a/.github/workflows/bin/find-missing-mirror-specs.py b/.github/workflows/bin/find-missing-mirror-specs.py deleted file mode 100644 index 3ae83a4f4c3..00000000000 --- a/.github/workflows/bin/find-missing-mirror-specs.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright Spack Project Developers. See COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""Print one ``pkg@=version`` spec per line for every checksummed package -version that is not yet present in the source mirror. - -Usage: spack python find-missing-mirror-specs.py - -```` contains one sha256 digest per line, obtained by listing the -content-addressed ``_source-cache/archive/`` prefix of the mirror. Only -URL-based versions with a ``sha256`` are considered, which matches the -content-addressed layout (``archive//.``) used by -``spack mirror create`` and mirrors the filtering done by -``spack repo show-version-updates --no-manual-packages --only-redistributable ---no-git-versions``. -""" - -import sys -from typing import Dict, List - -import spack.repo -import spack.spec -from spack.util import tty -from spack.version import StandardVersion - - -def main(sha256_file: str) -> None: - with open(sha256_file) as f: - # Store shas as a set / hash-table for faster key lookups - mirrored = {line.strip() for line in f if line.strip()} - - specs_to_output: List[spack.spec.Spec] = [] - - repo = spack.repo.PATH.get_repo("builtin") - - for pkg_cls in repo.all_package_classes(): - # Filter out manual packages - if pkg_cls.manual_download: - continue - - # Get all versions with checksums; no sha256 means not a - # content-addressed URL fetch (e.g. a git version) - version_to_checksum: Dict[StandardVersion, str] = { - version: version_dict["sha256"] - for version, version_dict in pkg_cls.versions.items() - if "sha256" in version_dict - } - - for version, sha256 in version_to_checksum.items(): - if sha256 in mirrored: - continue - version_spec = spack.spec.Spec(pkg_cls.name) - version_spec.constrain(f"@={version}") - specs_to_output.append(version_spec) - - # Filter out non-redistributable packages - specs_to_output = [ - spec for spec in specs_to_output if repo.get_pkg_class(spec.name).redistribute_source(spec) - ] - - # Output specs one per line for use by `spack mirror create` - # limit to a maximum of 100 specs at a time due to GitHub - # runner disk space limitations. Skipped specs will be - # retried on the next scheduled job. - specs_to_output_num = len(specs_to_output) - if specs_to_output_num > 100: - tty.warn(f"Limiting to first 100 missing specs. Detected {specs_to_output_num} missing.") - - for spec in specs_to_output[:100]: - print(spec) - - -if __name__ == "__main__": - main(sys.argv[1]) diff --git a/.github/workflows/sync-src-mirror.yml b/.github/workflows/sync-src-mirror.yml deleted file mode 100644 index 903d474d017..00000000000 --- a/.github/workflows/sync-src-mirror.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: sync-src-mirror - -# Nightly backfill of the source mirror: finds package versions missing from -# the mirror (e.g. because update-src-mirror failed or a fetch was flaky) and -# mirrors them. - -on: - schedule: - - cron: '0 3 * * *' # nightly at 03:00 UTC - workflow_dispatch: - -permissions: - id-token: write # Required for AWS OIDC authentication - contents: read - -jobs: - sync-mirror: - if: github.repository == 'spack/spack-packages' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Checkout Spack - uses: ./.github/actions/checkout-spack - with: - fetch-depth: 1 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Get digests already in the mirror - run: | - # Tarballs are content addressed as _source-cache/archive//.. - # Thus a single recursive listing of the archive/ prefix is a complete list - # of mirrored versions, patches, and package resources. Use awk and sed to strip - # out the shas from the filenames and save the shas to a file for later use. - aws s3 ls --recursive "s3://${{ secrets.S3_BUCKET_NAME }}/_source-cache/archive/" \ - | awk '{print $4}' \ - | sed -e 's|.*/||' -e 's|\..*||' \ - | sort -u > mirrored-shas.txt - echo "$(wc -l < mirrored-shas.txt) shas in the mirror" - - - name: Find shas in repository missing from mirror - id: find-missing - run: | - source spack-core/share/spack/setup-env.sh - - spack python .github/workflows/bin/find-missing-mirror-specs.py \ - mirrored-shas.txt > missing-specs.txt - - if [ ! -s missing-specs.txt ]; then - echo "Mirror is up to date" - echo "missing=false" >> $GITHUB_OUTPUT - exit 0 - fi - - echo "$(wc -l < missing-specs.txt) versions missing from the mirror:" - cat missing-specs.txt - echo "missing=true" >> $GITHUB_OUTPUT - - - name: Create source mirror - id: create-mirror - if: steps.find-missing.outputs.missing == 'true' - run: | - source spack-core/share/spack/setup-env.sh - - # Tolerate individual fetch failures (dead upstream URLs etc.) and - # upload whatever was mirrored successfully; anything that failed - # will simply be retried on the next nightly run. - spack -c concretizer:unify:false -c config:deprecated:true \ - mirror create -d src-mirror --jobs 8 --file missing-specs.txt \ - || echo "Some sources failed to fetch; uploading the rest" - - if [ -d src-mirror/_source-cache ]; then - echo "upload=true" >> $GITHUB_OUTPUT - else - echo "Nothing was mirrored" - echo "upload=false" >> $GITHUB_OUTPUT - fi - - - name: Upload to S3 - if: steps.create-mirror.outputs.upload == 'true' - run: | - aws s3 cp src-mirror/_source-cache s3://${{ secrets.S3_BUCKET_NAME }}/_source-cache/ \ - --no-overwrite \ - --recursive \ - --no-progress - echo "Successfully uploaded source mirror to S3"