diff --git a/.github/workflows/bin/sync-src-mirror.py b/.github/workflows/bin/sync-src-mirror.py new file mode 100755 index 00000000000..6def355eff8 --- /dev/null +++ b/.github/workflows/bin/sync-src-mirror.py @@ -0,0 +1,246 @@ +#!/usr/bin/env spack-python +# +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Sync the source mirror with the package repository -- without concretizing. + +Usage: sync-src-mirror.py [--upload-to s3://bucket] + +```` contains one sha256 digest per line, obtained by listing the +content-addressed ``_source-cache/archive/`` prefix of the mirror. + +This script: + 1. finds every source artifact (version tarball, resource, or patch) + that is missing from the mirror; + 2. Fetches it with spack's own fetch strategies that honor per-package + ``fetch_options``, mirrors, and checksum verification; and + 3. Uploads each verified artifact to the mirror with + ``aws s3 cp``. + +Uploads are one at a time, to bound disk usage. +Without ``--upload-to``, it just lists what is missing. + +This behaves like `spack mirror create --all`, in that all resources and +patches are included regardless of their ``when=`` conditions, since the mirror +should hold artifacts for every possible configuration. Only sha256-addressed +URL fetches are considered, which matches the content-addressed mirror +layout; git/svn/etc. versions have no place in ``_source-cache/archive``. + +Artifacts are stored at ``_source-cache/archive//[.]``, +computed with the same ``default_mirror_layout()`` spack itself uses. + +Individual fetch failures (dead upstream URLs etc.) are tolerated; they will +simply be retried on the next run. + +TODO: Parts of this should likely be integrated with `spack mirror create` +eventually. This exists in spack-packages because `spack mirror create --all` +currently concretizes specs when run from an environment, and it's hard to write +an environment to fetch only artifacts needed by certain package versions without +reconcretizing. Once that is done, replace this script. + +""" + +import argparse +from typing import Dict, List, Optional, Set, Tuple + +import spack.error +import spack.fetch_strategy +import spack.package_base +import spack.patch +import spack.repo +import spack.spec +import spack.stage +from spack.mirrors.layout import default_mirror_layout +from spack.util import tty +from spack.util.executable import Executable, which + +#: Cap on artifacts mirrored per run so a single nightly job is bounded. +#: Artifacts mirrored successfully drop out of the missing list, so anything +#: past the cap is picked up by subsequent runs. +MAX_ARTIFACTS = 1000 + +#: digest -> (mirror path, fetcher, human-readable label) +Entry = Tuple[str, spack.fetch_strategy.URLFetchStrategy, str] + + +def entry_for_fetcher( + fetcher: spack.fetch_strategy.FetchStrategy, + label: str, + mirrored: Set[str], + spec: Optional[spack.spec.Spec] = None, + extra_urls: Optional[List[str]] = None, +) -> Optional[Tuple[str, Entry]]: + """Return ``(digest, (mirror_path, fetcher, label))`` if ``fetcher`` is a + sha256-addressed URL fetch missing from the mirror, else ``None``.""" + if not isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy): + return None + + # Only sha256 digests: the content-addressed layout assumes them. Note + # that for compressed patches this is the *archive* sha256, which is what + # addresses the mirror entry. + digest = fetcher.digest + if not digest or len(digest) != 64 or digest in mirrored: + return None + + try: + # The alias argument is only used for the human-readable symlink, + # which we never create; digest_path is the content-addressed path. + layout = default_mirror_layout(fetcher, "unused", spec) + except spack.error.MirrorError as e: + tty.warn(str(e)) + return None + + # Fold any fallback URLs into the fetcher's mirrors so fetch() tries them + for url in extra_urls or (): + if url != fetcher.url and url not in fetcher.mirrors: + fetcher.mirrors.append(url) + + return digest, (layout.digest_path, fetcher, label) + + +def missing_artifacts(mirrored: Set[str]) -> Dict[str, Entry]: + """Map each missing sha256 to its mirror path, fetcher, and label.""" + entries: Dict[str, Entry] = {} + repo = spack.repo.PATH.get_repo("builtin") + + for pkg_cls in repo.all_package_classes(): + # Manual-download packages cannot be fetched by URL + if pkg_cls.manual_download: + continue + + try: + pkg = pkg_cls(spack.spec.Spec(pkg_cls.name)) + except Exception as e: + tty.warn(f"{pkg_cls.name}: could not instantiate package: {e}") + continue + + # Version tarballs. Restrict to versions with a sha256 up front; that + # skips git/manual versions cheaply and mirrors the filtering done by + # the content-addressed layout itself. + for version, version_dict in pkg_cls.versions.items(): + sha256 = version_dict.get("sha256") + if not isinstance(sha256, str) or sha256 in entries or sha256 in mirrored: + continue + + # Skip versions we may not redistribute (proprietary sources) + version_spec = spack.spec.Spec(f"{pkg_cls.name}@={version}") + if not pkg_cls.redistribute_source(version_spec): + continue + + try: + fetcher = spack.package_base.for_package_version(pkg, version) + # Fall back to any other URLs the package knows for this + # version (url_for_version, urls list, ...) + extra_urls = pkg.all_urls_for_version(version) + except Exception as e: + tty.warn(f"{pkg_cls.name}@{version}: could not determine URL: {e}") + continue + + entry = entry_for_fetcher( + fetcher, str(version_spec), mirrored, spec=version_spec, extra_urls=extra_urls + ) + if entry: + entries[entry[0]] = entry[1] + + # Resources, regardless of their when= conditions + for resource_list in pkg_cls.resources.values(): + for resource in resource_list: + entry = entry_for_fetcher( + resource.fetcher, f"{pkg_cls.name} resource {resource.name}", mirrored + ) + if entry: + entries.setdefault(entry[0], entry[1]) + + # Patches come from the repo's patch index rather than per-package + # ``patches`` attributes: the index also covers patches applied to + # dependencies via ``depends_on(..., patches=...)``, which spack looks up + # by sha256 from the ``patches=`` variant. Accessing the index builds the + # repo's data cache if needed. FilePatch (no ``url`` key) lives in the + # repo itself and needs no mirroring. Note that compressed patches are + # mirrored by their *archive* sha256, not the index key, which is the + # sha256 of the uncompressed patch. + for sha256, by_pkg in repo.get_patch_index().index.items(): + for patch_dict in by_pkg.values(): + if "url" not in patch_dict: + continue + try: + # the sha256 is removed from entries on write to save space, + # since it is the index key; add it back (see Patch.to_dict()) + patch_dict = dict(patch_dict, sha256=sha256) + patch = spack.patch.from_dict(patch_dict, repository=spack.repo.PATH) + except Exception as e: + tty.warn(f"could not read patch: {patch_dict.get('url')}: {e}") + continue + assert isinstance(patch, spack.patch.UrlPatch) + entry = entry_for_fetcher( + patch.fetcher(), f"{patch.owner} patch {patch.url}", mirrored + ) + if entry: + entries.setdefault(entry[0], entry[1]) + + return entries + + +def mirror_artifact( + path: str, fetcher: spack.fetch_strategy.FetchStrategy, s3_url: str, aws: Executable +) -> None: + """Fetch one artifact, verify its checksum, and upload it to the mirror. + + The stage (and the downloaded file with it) is destroyed on exit either + way, so disk usage stays bounded to one artifact at a time. + """ + with spack.stage.Stage(fetcher) as stage: + stage.fetch() + stage.check() + aws("s3", "cp", stage.archive_file, f"{s3_url}/{path}", "--no-overwrite", "--no-progress") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("sha256_file", help="file with one mirrored sha256 digest per line") + parser.add_argument( + "--upload-to", metavar="S3_URL", help="mirror url (s3://bucket); omit to just list" + ) + args = parser.parse_args() + + # fail fast if we can't upload before fetching anything + aws = which("aws", required=True) if args.upload_to else None + + with open(args.sha256_file) as f: + # Store shas as a set / hash-table for faster key lookups + mirrored = {line.strip() for line in f if line.strip()} + + entries = missing_artifacts(mirrored) + if not entries: + tty.msg("Mirror is up to date") + return + + if len(entries) > MAX_ARTIFACTS: + tty.warn( + f"Limiting to first {MAX_ARTIFACTS} missing artifacts. " + f"Detected {len(entries)} missing." + ) + + uploaded, failed = 0, 0 + for digest, (path, fetcher, label) in list(entries.items())[:MAX_ARTIFACTS]: + print(f"{label}: {path}") + if not args.upload_to: + continue + + try: + mirror_artifact(path, fetcher, args.upload_to.rstrip("/"), aws) + uploaded += 1 + except Exception as e: + tty.warn(f"could not mirror {label}: {e}") + failed += 1 + + if args.upload_to: + tty.msg(f"Uploaded {uploaded} artifacts to the mirror; {failed} failed") + else: + tty.msg(f"{len(entries)} artifacts missing from the mirror (listed only; no --upload-to)") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/sync-src-mirror.yml b/.github/workflows/sync-src-mirror.yml new file mode 100644 index 00000000000..71703b89adf --- /dev/null +++ b/.github/workflows/sync-src-mirror.yml @@ -0,0 +1,54 @@ +name: sync-src-mirror + +# Nightly backfill of the source mirror: finds version tarballs, resources, +# and patches missing from the mirror (e.g. because update-src-mirror failed +# or a fetch was flaky) and mirrors them. URLs and checksums are all known +# statically from package.py files, so nothing here concretizes. Fetching +# and checksum verification use spack's own fetch strategies. + +on: + schedule: + - cron: '0 3 * * *' # nightly at 03:00 UTC + workflow_dispatch: + +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read + +jobs: + sync-mirror: + if: github.repository == 'spack/spack-packages' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Checkout Spack + uses: ./.github/actions/checkout-spack + with: + fetch-depth: 1 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Get digests already in the mirror + run: | + # Tarballs are content addressed as _source-cache/archive//.. + # Thus a single recursive listing of the archive/ prefix is a complete list + # of mirrored versions, patches, and package resources. Use awk and sed to strip + # out the shas from the filenames and save the shas to a file for later use. + aws s3 ls --recursive "s3://${{ secrets.S3_BUCKET_NAME }}/_source-cache/archive/" \ + | awk '{print $4}' \ + | sed -e 's|.*/||' -e 's|\..*||' \ + | sort -u > mirrored-shas.txt + echo "$(wc -l < mirrored-shas.txt) shas in the mirror" + + - name: Sync missing artifacts to the mirror + run: | + source spack-core/share/spack/setup-env.sh + + .github/workflows/bin/sync-src-mirror.py \ + mirrored-shas.txt --upload-to "s3://${{ secrets.S3_BUCKET_NAME }}"