From 67124179ffb6452928d4cda493e84615769737c8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 26 Aug 2026 17:31:17 -0700 Subject: [PATCH 1/2] ci: sync source mirror statically, without concretizing Re-adds the nightly source mirror sync from #6069 (reverted in #6245). Nothing about mirroring sources requires concretization: URLs and sha256 checksums for version tarballs, resources, and patches are all known statically from `package.py` files. The `find-missing-mirror-artifacts.py` script here: 1. lists every sha256-addressed artifact in the builtin repo: - version tarballs - resources - URL patches (in packages and from dependencies) while skipping manual-download and non-redistributable packages; 2. compares digests against a listing of the mirror's content-addressed `_source-cache/archive/` prefix; and 3. emits one TSV line per missing artifact with its: - `sha256` - exact mirror path, computed with Spack's `default_mirror_layout()` - its candidate URLs in order of preference. A GitHub Actions workflow then downloads each artifact with curl, verifies the sha256, and uploads artifacts to S3 one at a time. disk usage is bounded and individual fetch failures only skip that artifact until the next nightly run. Co-authored-by: Alec Scott Assisted-by: Claude Signed-off-by: Todd Gamblin --- .../bin/find-missing-mirror-artifacts.py | 183 ++++++++++++++++++ .github/workflows/sync-src-mirror.yml | 107 ++++++++++ 2 files changed, 290 insertions(+) create mode 100644 .github/workflows/bin/find-missing-mirror-artifacts.py create mode 100644 .github/workflows/sync-src-mirror.yml diff --git a/.github/workflows/bin/find-missing-mirror-artifacts.py b/.github/workflows/bin/find-missing-mirror-artifacts.py new file mode 100644 index 00000000000..8f51c17b29e --- /dev/null +++ b/.github/workflows/bin/find-missing-mirror-artifacts.py @@ -0,0 +1,183 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Print one line per source artifact (version tarball, resource, or patch) +that is missing from the source mirror -- without concretizing anything. + +Usage: spack python find-missing-mirror-artifacts.py + +```` contains one sha256 digest per line, obtained by listing the +content-addressed ``_source-cache/archive/`` prefix of the mirror. + +Each output line is tab-separated:: + + TAB TAB [TAB ...] + +``mirror-path`` is the content-addressed location in the mirror +(``_source-cache/archive//[.]``), computed with the +same ``default_mirror_layout()`` spack itself uses, so files uploaded to these +paths are found by ``spack fetch``. The URLs are candidate download locations +in order of preference. + +Unlike ``spack mirror create``, nothing here requires concretization: URLs and +checksums for version tarballs, resources, and patches are all known +statically from package.py files. Conditional resources and patches are +included regardless of their ``when=`` conditions, since the mirror should +hold artifacts for every possible configuration. Only sha256-addressed URL +fetches are considered, which matches the content-addressed mirror layout; +git/svn/etc. versions have no place in ``_source-cache/archive``. +""" + +import sys +from typing import Dict, List, Optional, Set, Tuple + +import spack.error +import spack.fetch_strategy +import spack.package_base +import spack.patch +import spack.repo +import spack.spec +from spack.mirrors.layout import default_mirror_layout +from spack.util import tty + +#: Cap on artifacts emitted per run so a single nightly job is bounded. +#: Artifacts mirrored successfully drop out of the missing list, so anything +#: past the cap is picked up by subsequent runs. +MAX_ARTIFACTS = 1000 + +#: digest -> (mirror path, candidate urls) +Entry = Tuple[str, List[str]] + + +def entry_for_fetcher( + fetcher: spack.fetch_strategy.FetchStrategy, + mirrored: Set[str], + spec: Optional[spack.spec.Spec] = None, + extra_urls: Optional[List[str]] = None, +) -> Optional[Tuple[str, Entry]]: + """Return ``(digest, (mirror_path, urls))`` if ``fetcher`` is a + sha256-addressed URL fetch missing from the mirror, else ``None``.""" + if not isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy): + return None + + # Only sha256 digests: the content-addressed layout and the workflow's + # sha256sum verification both assume them. Note that for compressed + # patches this is the *archive* sha256, which is what addresses the + # mirror entry. + digest = fetcher.digest + if not digest or len(digest) != 64 or digest in mirrored: + return None + + try: + # The alias argument is only used for the human-readable symlink, + # which we never create; digest_path is the content-addressed path. + layout = default_mirror_layout(fetcher, "unused", spec) + except spack.error.MirrorError as e: + tty.warn(str(e)) + return None + + urls = list(fetcher.candidate_urls) + for url in extra_urls or (): + if url not in urls: + urls.append(url) + + return digest, (layout.digest_path, urls) + + +def missing_artifacts(mirrored: Set[str]) -> Dict[str, Entry]: + """Map each missing sha256 to its mirror path and candidate URLs.""" + entries: Dict[str, Entry] = {} + repo = spack.repo.PATH.get_repo("builtin") + + for pkg_cls in repo.all_package_classes(): + # Manual-download packages cannot be fetched by URL + if pkg_cls.manual_download: + continue + + try: + pkg = pkg_cls(spack.spec.Spec(pkg_cls.name)) + except Exception as e: + tty.warn(f"{pkg_cls.name}: could not instantiate package: {e}") + continue + + # Version tarballs. Restrict to versions with a sha256 up front; that + # skips git/manual versions cheaply and mirrors the filtering done by + # the content-addressed layout itself. + for version, version_dict in pkg_cls.versions.items(): + sha256 = version_dict.get("sha256") + if not isinstance(sha256, str) or sha256 in entries or sha256 in mirrored: + continue + + # Skip versions we may not redistribute (proprietary sources) + version_spec = spack.spec.Spec(f"{pkg_cls.name}@={version}") + if not pkg_cls.redistribute_source(version_spec): + continue + + try: + fetcher = spack.package_base.for_package_version(pkg, version) + # Fall back to any other URLs the package knows for this + # version (url_for_version, urls list, ...) + extra_urls = pkg.all_urls_for_version(version) + except Exception as e: + tty.warn(f"{pkg_cls.name}@{version}: could not determine URL: {e}") + continue + + entry = entry_for_fetcher(fetcher, mirrored, spec=version_spec, extra_urls=extra_urls) + if entry: + entries[entry[0]] = entry[1] + + # Resources, regardless of their when= conditions + for resource_list in pkg_cls.resources.values(): + for resource in resource_list: + entry = entry_for_fetcher(resource.fetcher, mirrored) + if entry: + entries.setdefault(entry[0], entry[1]) + + # Patches come from the repo's patch index rather than per-package + # ``patches`` attributes: the index also covers patches applied to + # dependencies via ``depends_on(..., patches=...)``, which spack looks up + # by sha256 from the ``patches=`` variant. Accessing the index builds the + # repo's data cache if needed. FilePatch (no ``url`` key) lives in the + # repo itself and needs no mirroring. Note that compressed patches are + # mirrored by their *archive* sha256, not the index key, which is the + # sha256 of the uncompressed patch. + for sha256, by_pkg in repo.get_patch_index().index.items(): + for patch_dict in by_pkg.values(): + if "url" not in patch_dict: + continue + try: + # the sha256 is removed from entries on write to save space, + # since it is the index key; add it back (see Patch.to_dict()) + patch_dict = dict(patch_dict, sha256=sha256) + patch = spack.patch.from_dict(patch_dict, repository=spack.repo.PATH) + except Exception as e: + tty.warn(f"could not read patch: {patch_dict.get('url')}: {e}") + continue + assert isinstance(patch, spack.patch.UrlPatch) + entry = entry_for_fetcher(patch.fetcher(), mirrored) + if entry: + entries.setdefault(entry[0], entry[1]) + + return entries + + +def main(sha256_file: str) -> None: + with open(sha256_file) as f: + # Store shas as a set / hash-table for faster key lookups + mirrored = {line.strip() for line in f if line.strip()} + + entries = missing_artifacts(mirrored) + + if len(entries) > MAX_ARTIFACTS: + tty.warn( + f"Limiting to first {MAX_ARTIFACTS} missing artifacts. " + f"Detected {len(entries)} missing." + ) + + for digest, (path, urls) in list(entries.items())[:MAX_ARTIFACTS]: + print("\t".join([digest, path, *urls])) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/.github/workflows/sync-src-mirror.yml b/.github/workflows/sync-src-mirror.yml new file mode 100644 index 00000000000..0afe00f2076 --- /dev/null +++ b/.github/workflows/sync-src-mirror.yml @@ -0,0 +1,107 @@ +name: sync-src-mirror + +# Nightly backfill of the source mirror: finds version tarballs, resources, +# and patches missing from the mirror (e.g. because update-src-mirror failed +# or a fetch was flaky) and mirrors them. URLs and checksums are all known +# statically from package.py files, so nothing here concretizes. + +on: + schedule: + - cron: '0 3 * * *' # nightly at 03:00 UTC + workflow_dispatch: + +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read + +jobs: + sync-mirror: + if: github.repository == 'spack/spack-packages' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Checkout Spack + uses: ./.github/actions/checkout-spack + with: + fetch-depth: 1 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Get digests already in the mirror + run: | + # Tarballs are content addressed as _source-cache/archive//.. + # Thus a single recursive listing of the archive/ prefix is a complete list + # of mirrored versions, patches, and package resources. Use awk and sed to strip + # out the shas from the filenames and save the shas to a file for later use. + aws s3 ls --recursive "s3://${{ secrets.S3_BUCKET_NAME }}/_source-cache/archive/" \ + | awk '{print $4}' \ + | sed -e 's|.*/||' -e 's|\..*||' \ + | sort -u > mirrored-shas.txt + echo "$(wc -l < mirrored-shas.txt) shas in the mirror" + + - name: Find artifacts in repository missing from mirror + id: find-missing + run: | + source spack-core/share/spack/setup-env.sh + + spack python .github/workflows/bin/find-missing-mirror-artifacts.py \ + mirrored-shas.txt > missing-artifacts.tsv + + if [ ! -s missing-artifacts.tsv ]; then + echo "Mirror is up to date" + echo "missing=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "$(wc -l < missing-artifacts.tsv) artifacts missing from the mirror:" + cut -f2 missing-artifacts.tsv + echo "missing=true" >> $GITHUB_OUTPUT + + - name: Download, verify, and upload missing artifacts + if: steps.find-missing.outputs.missing == 'true' + run: | + # Each line is: TAB TAB [TAB ...]. + # Download with curl (same flags spack uses: -f -L -sS), verify the + # sha256, and upload each artifact individually so disk usage stays + # bounded. Individual fetch failures (dead upstream URLs etc.) are + # tolerated; they will simply be retried on the next nightly run. + uploaded=0 + failed=0 + mkdir -p artifact-tmp + + while IFS=$'\t' read -r sha path url_list; do + file="artifact-tmp/${path##*/}" + ok=false + + while IFS= read -r url; do + rm -f "$file" + if curl -fLsS --retry 2 --connect-timeout 30 --max-time 1800 \ + -o "$file" "$url"; then + if echo "$sha $file" | sha256sum --check --quiet -; then + ok=true + break + fi + echo "checksum mismatch for $url" + else + echo "failed to fetch $url" + fi + done < <(tr '\t' '\n' <<< "$url_list") + + if $ok; then + aws s3 cp "$file" "s3://${{ secrets.S3_BUCKET_NAME }}/$path" \ + --no-overwrite --no-progress + uploaded=$((uploaded + 1)) + else + echo "could not mirror $path" + failed=$((failed + 1)) + fi + rm -f "$file" + done < missing-artifacts.tsv + + echo "Uploaded $uploaded artifacts to the mirror; $failed failed" From 5f1353798662629306e91cfd376e09e9edd55cee Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 26 Aug 2026 23:13:30 -0700 Subject: [PATCH 2/2] sync-src-mirror: fetch with spack instead of curl Merge the download/verify/upload loop into the finder script and fetch with spack's own fetch strategies via Stage.fetch()/check(). This honors per-package fetch_options, reuses spack's checksum and redirect handling, and drops the TSV/bash handoff. Uploads still go through `aws s3 cp`, one artifact at a time, so disk stays bounded. Without --upload-to, the script just lists missing artifacts, which makes it easy to run locally. Assisted-by: Claude Signed-off-by: Todd Gamblin --- ...mirror-artifacts.py => sync-src-mirror.py} | 141 +++++++++++++----- .github/workflows/sync-src-mirror.yml | 63 +------- 2 files changed, 107 insertions(+), 97 deletions(-) rename .github/workflows/bin/{find-missing-mirror-artifacts.py => sync-src-mirror.py} (52%) mode change 100644 => 100755 diff --git a/.github/workflows/bin/find-missing-mirror-artifacts.py b/.github/workflows/bin/sync-src-mirror.py old mode 100644 new mode 100755 similarity index 52% rename from .github/workflows/bin/find-missing-mirror-artifacts.py rename to .github/workflows/bin/sync-src-mirror.py index 8f51c17b29e..6def355eff8 --- a/.github/workflows/bin/find-missing-mirror-artifacts.py +++ b/.github/workflows/bin/sync-src-mirror.py @@ -1,35 +1,48 @@ +#!/usr/bin/env spack-python +# # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -"""Print one line per source artifact (version tarball, resource, or patch) -that is missing from the source mirror -- without concretizing anything. +"""Sync the source mirror with the package repository -- without concretizing. -Usage: spack python find-missing-mirror-artifacts.py +Usage: sync-src-mirror.py [--upload-to s3://bucket] ```` contains one sha256 digest per line, obtained by listing the content-addressed ``_source-cache/archive/`` prefix of the mirror. -Each output line is tab-separated:: +This script: + 1. finds every source artifact (version tarball, resource, or patch) + that is missing from the mirror; + 2. Fetches it with spack's own fetch strategies that honor per-package + ``fetch_options``, mirrors, and checksum verification; and + 3. Uploads each verified artifact to the mirror with + ``aws s3 cp``. + +Uploads are one at a time, to bound disk usage. +Without ``--upload-to``, it just lists what is missing. + +This behaves like `spack mirror create --all`, in that all resources and +patches are included regardless of their ``when=`` conditions, since the mirror +should hold artifacts for every possible configuration. Only sha256-addressed +URL fetches are considered, which matches the content-addressed mirror +layout; git/svn/etc. versions have no place in ``_source-cache/archive``. + +Artifacts are stored at ``_source-cache/archive//[.]``, +computed with the same ``default_mirror_layout()`` spack itself uses. - TAB TAB [TAB ...] +Individual fetch failures (dead upstream URLs etc.) are tolerated; they will +simply be retried on the next run. -``mirror-path`` is the content-addressed location in the mirror -(``_source-cache/archive//[.]``), computed with the -same ``default_mirror_layout()`` spack itself uses, so files uploaded to these -paths are found by ``spack fetch``. The URLs are candidate download locations -in order of preference. +TODO: Parts of this should likely be integrated with `spack mirror create` +eventually. This exists in spack-packages because `spack mirror create --all` +currently concretizes specs when run from an environment, and it's hard to write +an environment to fetch only artifacts needed by certain package versions without +reconcretizing. Once that is done, replace this script. -Unlike ``spack mirror create``, nothing here requires concretization: URLs and -checksums for version tarballs, resources, and patches are all known -statically from package.py files. Conditional resources and patches are -included regardless of their ``when=`` conditions, since the mirror should -hold artifacts for every possible configuration. Only sha256-addressed URL -fetches are considered, which matches the content-addressed mirror layout; -git/svn/etc. versions have no place in ``_source-cache/archive``. """ -import sys +import argparse from typing import Dict, List, Optional, Set, Tuple import spack.error @@ -38,33 +51,35 @@ import spack.patch import spack.repo import spack.spec +import spack.stage from spack.mirrors.layout import default_mirror_layout from spack.util import tty +from spack.util.executable import Executable, which -#: Cap on artifacts emitted per run so a single nightly job is bounded. +#: Cap on artifacts mirrored per run so a single nightly job is bounded. #: Artifacts mirrored successfully drop out of the missing list, so anything #: past the cap is picked up by subsequent runs. MAX_ARTIFACTS = 1000 -#: digest -> (mirror path, candidate urls) -Entry = Tuple[str, List[str]] +#: digest -> (mirror path, fetcher, human-readable label) +Entry = Tuple[str, spack.fetch_strategy.URLFetchStrategy, str] def entry_for_fetcher( fetcher: spack.fetch_strategy.FetchStrategy, + label: str, mirrored: Set[str], spec: Optional[spack.spec.Spec] = None, extra_urls: Optional[List[str]] = None, ) -> Optional[Tuple[str, Entry]]: - """Return ``(digest, (mirror_path, urls))`` if ``fetcher`` is a + """Return ``(digest, (mirror_path, fetcher, label))`` if ``fetcher`` is a sha256-addressed URL fetch missing from the mirror, else ``None``.""" if not isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy): return None - # Only sha256 digests: the content-addressed layout and the workflow's - # sha256sum verification both assume them. Note that for compressed - # patches this is the *archive* sha256, which is what addresses the - # mirror entry. + # Only sha256 digests: the content-addressed layout assumes them. Note + # that for compressed patches this is the *archive* sha256, which is what + # addresses the mirror entry. digest = fetcher.digest if not digest or len(digest) != 64 or digest in mirrored: return None @@ -77,16 +92,16 @@ def entry_for_fetcher( tty.warn(str(e)) return None - urls = list(fetcher.candidate_urls) + # Fold any fallback URLs into the fetcher's mirrors so fetch() tries them for url in extra_urls or (): - if url not in urls: - urls.append(url) + if url != fetcher.url and url not in fetcher.mirrors: + fetcher.mirrors.append(url) - return digest, (layout.digest_path, urls) + return digest, (layout.digest_path, fetcher, label) def missing_artifacts(mirrored: Set[str]) -> Dict[str, Entry]: - """Map each missing sha256 to its mirror path and candidate URLs.""" + """Map each missing sha256 to its mirror path, fetcher, and label.""" entries: Dict[str, Entry] = {} repo = spack.repo.PATH.get_repo("builtin") @@ -123,14 +138,18 @@ def missing_artifacts(mirrored: Set[str]) -> Dict[str, Entry]: tty.warn(f"{pkg_cls.name}@{version}: could not determine URL: {e}") continue - entry = entry_for_fetcher(fetcher, mirrored, spec=version_spec, extra_urls=extra_urls) + entry = entry_for_fetcher( + fetcher, str(version_spec), mirrored, spec=version_spec, extra_urls=extra_urls + ) if entry: entries[entry[0]] = entry[1] # Resources, regardless of their when= conditions for resource_list in pkg_cls.resources.values(): for resource in resource_list: - entry = entry_for_fetcher(resource.fetcher, mirrored) + entry = entry_for_fetcher( + resource.fetcher, f"{pkg_cls.name} resource {resource.name}", mirrored + ) if entry: entries.setdefault(entry[0], entry[1]) @@ -155,19 +174,48 @@ def missing_artifacts(mirrored: Set[str]) -> Dict[str, Entry]: tty.warn(f"could not read patch: {patch_dict.get('url')}: {e}") continue assert isinstance(patch, spack.patch.UrlPatch) - entry = entry_for_fetcher(patch.fetcher(), mirrored) + entry = entry_for_fetcher( + patch.fetcher(), f"{patch.owner} patch {patch.url}", mirrored + ) if entry: entries.setdefault(entry[0], entry[1]) return entries -def main(sha256_file: str) -> None: - with open(sha256_file) as f: +def mirror_artifact( + path: str, fetcher: spack.fetch_strategy.FetchStrategy, s3_url: str, aws: Executable +) -> None: + """Fetch one artifact, verify its checksum, and upload it to the mirror. + + The stage (and the downloaded file with it) is destroyed on exit either + way, so disk usage stays bounded to one artifact at a time. + """ + with spack.stage.Stage(fetcher) as stage: + stage.fetch() + stage.check() + aws("s3", "cp", stage.archive_file, f"{s3_url}/{path}", "--no-overwrite", "--no-progress") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("sha256_file", help="file with one mirrored sha256 digest per line") + parser.add_argument( + "--upload-to", metavar="S3_URL", help="mirror url (s3://bucket); omit to just list" + ) + args = parser.parse_args() + + # fail fast if we can't upload before fetching anything + aws = which("aws", required=True) if args.upload_to else None + + with open(args.sha256_file) as f: # Store shas as a set / hash-table for faster key lookups mirrored = {line.strip() for line in f if line.strip()} entries = missing_artifacts(mirrored) + if not entries: + tty.msg("Mirror is up to date") + return if len(entries) > MAX_ARTIFACTS: tty.warn( @@ -175,9 +223,24 @@ def main(sha256_file: str) -> None: f"Detected {len(entries)} missing." ) - for digest, (path, urls) in list(entries.items())[:MAX_ARTIFACTS]: - print("\t".join([digest, path, *urls])) + uploaded, failed = 0, 0 + for digest, (path, fetcher, label) in list(entries.items())[:MAX_ARTIFACTS]: + print(f"{label}: {path}") + if not args.upload_to: + continue + + try: + mirror_artifact(path, fetcher, args.upload_to.rstrip("/"), aws) + uploaded += 1 + except Exception as e: + tty.warn(f"could not mirror {label}: {e}") + failed += 1 + + if args.upload_to: + tty.msg(f"Uploaded {uploaded} artifacts to the mirror; {failed} failed") + else: + tty.msg(f"{len(entries)} artifacts missing from the mirror (listed only; no --upload-to)") if __name__ == "__main__": - main(sys.argv[1]) + main() diff --git a/.github/workflows/sync-src-mirror.yml b/.github/workflows/sync-src-mirror.yml index 0afe00f2076..71703b89adf 100644 --- a/.github/workflows/sync-src-mirror.yml +++ b/.github/workflows/sync-src-mirror.yml @@ -3,7 +3,8 @@ name: sync-src-mirror # Nightly backfill of the source mirror: finds version tarballs, resources, # and patches missing from the mirror (e.g. because update-src-mirror failed # or a fetch was flaky) and mirrors them. URLs and checksums are all known -# statically from package.py files, so nothing here concretizes. +# statically from package.py files, so nothing here concretizes. Fetching +# and checksum verification use spack's own fetch strategies. on: schedule: @@ -45,63 +46,9 @@ jobs: | sort -u > mirrored-shas.txt echo "$(wc -l < mirrored-shas.txt) shas in the mirror" - - name: Find artifacts in repository missing from mirror - id: find-missing + - name: Sync missing artifacts to the mirror run: | source spack-core/share/spack/setup-env.sh - spack python .github/workflows/bin/find-missing-mirror-artifacts.py \ - mirrored-shas.txt > missing-artifacts.tsv - - if [ ! -s missing-artifacts.tsv ]; then - echo "Mirror is up to date" - echo "missing=false" >> $GITHUB_OUTPUT - exit 0 - fi - - echo "$(wc -l < missing-artifacts.tsv) artifacts missing from the mirror:" - cut -f2 missing-artifacts.tsv - echo "missing=true" >> $GITHUB_OUTPUT - - - name: Download, verify, and upload missing artifacts - if: steps.find-missing.outputs.missing == 'true' - run: | - # Each line is: TAB TAB [TAB ...]. - # Download with curl (same flags spack uses: -f -L -sS), verify the - # sha256, and upload each artifact individually so disk usage stays - # bounded. Individual fetch failures (dead upstream URLs etc.) are - # tolerated; they will simply be retried on the next nightly run. - uploaded=0 - failed=0 - mkdir -p artifact-tmp - - while IFS=$'\t' read -r sha path url_list; do - file="artifact-tmp/${path##*/}" - ok=false - - while IFS= read -r url; do - rm -f "$file" - if curl -fLsS --retry 2 --connect-timeout 30 --max-time 1800 \ - -o "$file" "$url"; then - if echo "$sha $file" | sha256sum --check --quiet -; then - ok=true - break - fi - echo "checksum mismatch for $url" - else - echo "failed to fetch $url" - fi - done < <(tr '\t' '\n' <<< "$url_list") - - if $ok; then - aws s3 cp "$file" "s3://${{ secrets.S3_BUCKET_NAME }}/$path" \ - --no-overwrite --no-progress - uploaded=$((uploaded + 1)) - else - echo "could not mirror $path" - failed=$((failed + 1)) - fi - rm -f "$file" - done < missing-artifacts.tsv - - echo "Uploaded $uploaded artifacts to the mirror; $failed failed" + .github/workflows/bin/sync-src-mirror.py \ + mirrored-shas.txt --upload-to "s3://${{ secrets.S3_BUCKET_NAME }}"