diff --git a/.github/workflows/update-inputdata-manifest.yml b/.github/workflows/update-inputdata-manifest.yml new file mode 100644 index 0000000..46a147c --- /dev/null +++ b/.github/workflows/update-inputdata-manifest.yml @@ -0,0 +1,111 @@ +name: update-inputdata-manifest + +on: + schedule: + - cron: "0 12 * * 1" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: update-inputdata-manifest + cancel-in-progress: false + +jobs: + regenerate: + runs-on: ubuntu-22.04 + timeout-minutes: 240 + container: + image: ghcr.io/e3sm-project/containers-ghci:main + steps: + - name: Checkout containers + uses: actions/checkout@v4 + with: + submodules: recursive + show-progress: false + persist-credentials: false + + - name: Checkout E3SM + uses: actions/checkout@v4 + with: + repository: E3SM-Project/E3SM + path: E3SM + ref: master + submodules: recursive + show-progress: false + persist-credentials: false + + - name: Regenerate inputdata manifests + env: + NETCDF_C_ROOT: /usr + NETCDF_FORTRAN_ROOT: /usr + PARALLEL_NETCDF_ROOT: /usr + run: | + set -euo pipefail + mkdir -p /tmp/e3sm-inputdata-cases /tmp/e3sm-inputdata-output + python3 -m tools.inputdata.generate_inputdata_manifest \ + --e3sm-root E3SM \ + --test-root /tmp/e3sm-inputdata-cases \ + --output-root /tmp/e3sm-inputdata-output \ + --state-out inputdata/e3sm-workflow-state.json \ + --files-out inputdata/files.txt \ + --standalone-files-out inputdata/files-standalone.txt \ + --missing-files-out inputdata/files-missing.txt \ + --provenance-out inputdata/provenance.json \ + --verbose + + - name: Detect manifest changes + id: detect + run: | + set -euo pipefail + files=( + inputdata/files.txt + inputdata/files-standalone.txt + inputdata/files-missing.txt + ) + + if git diff --quiet -- "${files[@]}"; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Build PR metadata + if: steps.detect.outputs.changed == 'true' + id: metadata + run: | + set -euo pipefail + + stamp="$(date -u +%Y%m%d)" + branch="bot/update-inputdata-manifest-${stamp}-${GITHUB_RUN_ID}" + echo "branch=${branch}" >> "$GITHUB_OUTPUT" + + summary_path="/tmp/inputdata-pr-summary.md" + + python3 -m tools.inputdata.build_pr_metadata \ + --output-path "${summary_path}" \ + --state-path inputdata/e3sm-workflow-state.json + + echo "body_path=${summary_path}" >> "$GITHUB_OUTPUT" + + - name: Create pull request + if: steps.detect.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v6 + with: + branch: ${{ steps.metadata.outputs.branch }} + delete-branch: true + base: main + title: "Update E3SM inputdata manifest" + commit-message: "Update inputdata manifest from E3SM workflow tests" + body-path: ${{ steps.metadata.outputs.body_path }} + add-paths: | + inputdata/files.txt + inputdata/files-standalone.txt + inputdata/files-missing.txt + inputdata/e3sm-workflow-state.json + inputdata/provenance.json + labels: | + automated + inputdata \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..d657dcc --- /dev/null +++ b/README.md @@ -0,0 +1,126 @@ +# E3SM Containers Repository + +This repository contains Docker build contexts and automation for container +images used by E3SM CI and related workflows. + +## Repository Layout + +- `ghci/`: Base GHCI CI container build context. +- `standalone-ghci/`: Standalone GHCI container build context. +- `inputdata/`: Inputdata container build context and manifest files. +- `e3sm-diags-test-data/`: E3SM diags test-data image build context. +- `tools/inputdata/`: Inputdata manifest generation and support utilities. +- `.github/workflows/`: GitHub Actions workflows for build/publish and + manifest automation. + +## Workflow Summary + +All workflows live in `.github/workflows`. + +### `ghci.yaml` + +- Purpose: Build the `ghcr.io//-ghci` image. +- Triggers: + - `pull_request` to `main` when `ghci/**` or workflow file changes. + - `push` to `main` on same paths. + - tags matching `ghci-*`. +- Behavior: + - Always builds on PRs and pushes. + - Pushes image only when event is not `pull_request`. + +### `standalone-ghci.yaml` + +- Purpose: Build the `ghcr.io//-standalone-ghci` image. +- Triggers: + - `pull_request` to `main` when `standalone-ghci/**` or workflow file + changes. + - `push` to `main` on same paths. + - tags matching `standalone-ghci-*`. +- Behavior: + - Uses `ubuntu-22.04-arm` runner matrix. + - Pushes image only when event is not `pull_request`. + +### `e3sm-diags-test-data.yaml` + +- Purpose: Build the `ghcr.io//-e3sm-diags-test-data` image. +- Triggers: + - `merge_group` to `main`. + - `pull_request` to `main` when `e3sm-diags-test-data/**` or workflow file + changes. + - `push` to `main` on same paths. + - tags matching `e3sm-diags-test-data-*`. +- Behavior: + - Pushes image only when event is not `pull_request`. + +### `inputdata.yaml` + +- Purpose: Build inputdata images from manifest files in `inputdata/`. +- Triggers: + - `pull_request` to `main` when `inputdata/**` or workflow file changes. + - `push` to `main` on same paths. + - tags matching `inputdata-*`. +- Behavior: + - Matrix builds two variants: + - `files` from `inputdata/files.txt`. + - `files-standalone` from `inputdata/files-standalone.txt`. + - Pushes images only when event is not `pull_request`. + +### `update-inputdata-manifest.yml` + +- Purpose: Regenerate inputdata manifests from E3SM workflows and open a PR + when generated outputs change. +- Triggers: + - Weekly schedule (`cron: 0 12 * * 1`). + - Manual `workflow_dispatch`. +- Behavior: + - Checks out this repository and `E3SM-Project/E3SM` at `master`. + - Runs `python3 -m tools.inputdata.generate_inputdata_manifest`. + - Opens a PR only when these payload-defining files change: + - `inputdata/files.txt` + - `inputdata/files-standalone.txt` + - `inputdata/files-missing.txt` + - If a PR is opened, it also includes related metadata files: + - `inputdata/e3sm-workflow-state.json` + - `inputdata/provenance.json` + - If no changes: exits without opening a PR. + - If changes exist: uses `peter-evans/create-pull-request` to open a real + PR with generated metadata. + +## How Workflows Fit Together + +1. `update-inputdata-manifest.yml` is the source-of-truth updater for + inputdata manifest content. +2. When it opens a PR that modifies files under `inputdata/`, + `inputdata.yaml` runs automatically on that PR and validates container builds + without pushing images. +3. After the PR is reviewed and merged to `main`, `inputdata.yaml` runs again + on the `push` event and publishes updated inputdata images to GHCR. +4. `ghci.yaml`, `standalone-ghci.yaml`, and `e3sm-diags-test-data.yaml` are + independent image pipelines and do not depend on inputdata-manifest updates. + +In short: one workflow updates inputdata manifests, a different workflow builds +and publishes inputdata images from those manifests. + +## Common Trigger Pattern + +Most image workflows follow the same model: + +- Build on PRs for validation. +- Publish on `push` to `main` or on release-style tags. +- Scope execution with `paths` filters to avoid unrelated builds. + +## For New Contributors + +If you are updating manifest generation logic: + +1. Change code under `tools/inputdata/`. +2. Run the generator locally if needed. +3. Use `workflow_dispatch` on `update-inputdata-manifest.yml` to test the + automation path. + +If you are updating container images directly: + +1. Modify the corresponding directory (`ghci/`, `standalone-ghci/`, + `inputdata/`, or `e3sm-diags-test-data/`). +2. Open a PR and confirm the matching workflow build succeeds. +3. Merge to `main` to publish new images. \ No newline at end of file diff --git a/inputdata/files-missing.txt b/inputdata/files-missing.txt new file mode 100644 index 0000000..e69de29 diff --git a/readme b/readme deleted file mode 100644 index bc2e652..0000000 --- a/readme +++ /dev/null @@ -1 +0,0 @@ -Container recipes for the E3SM project diff --git a/tools/inputdata/README.md b/tools/inputdata/README.md new file mode 100644 index 0000000..13d4d48 --- /dev/null +++ b/tools/inputdata/README.md @@ -0,0 +1,56 @@ +# Inputdata Generator + +`tools/inputdata` contains the generator that builds the list of inputdata files required for Github +Continuous Integration tests. The purpose is to produce the files that should be included in the CI +Docker container, avoiding the need for them to be downloaded during CI. + +## What it does + +The main entry point is `python3 -m tools.inputdata.generate_inputdata_manifest`. It scans an E3SM checkout, extracts test names from workflow files, runs CIME `create_test` for full-model cases, and writes manifest and provenance files that describe the input files referenced by those tests. + +It also performs a best-effort standalone EAMxx scan so the generated manifests can include input paths referenced outside of the full-model CIME flow. + +## How it works + +The generator uses a small pipeline: + +1. `tools.inputdata.lib.workflow_parser` reads `.github/workflows/*.yml` and `.yaml` files from the E3SM tree. +2. It collects full-model test names from `create_test` lines and matrix definitions, and records standalone EAMxx references when present. +3. `tools.inputdata.lib.cime_manifest.run_create_test` invokes `cime/scripts/create_test` for each full-model test in a setup-only mode (`--no-build`, `--no-run`, `--namelists-only`). That creates the case metadata and Buildconf files needed to discover input lists, but it does not compile or execute the test case, and it avoids actually pulling model input data during the run. +4. `tools.inputdata.lib.url_normalize` converts discovered local paths into public inputdata URLs when they point at supported data. +5. `tools.inputdata.lib.inputdata_validation.validate_inputdata_reference` checks the candidate URL with a lightweight remote existence probe without downloading the file. It tries the E3SM public inputdata server first, then the CESM mirror. When both endpoints definitively return not-found responses, the entry is recorded as missing; when the servers cannot be verified cleanly, the entry is kept but marked unverified in provenance. +6. The generator writes the final outputs in a stable order so repeated runs produce the same files. + +Some `create_test` invocations can still print `FAIL` during setup. In this workflow, that usually means CIME could not finish case setup for that particular test, so the generator records the failure as a warning and moves on to the next test instead of treating it as a fatal error for the whole manifest run. Usually, the needed information is discovered despite this. + +## Inputs + +The command requires: + +- `--e3sm-root`: path to the E3SM checkout +- `--test-root`: directory where CIME test cases are created +- `--output-root`: directory where generated artifacts and temporary outputs are written +- `--state-out`, `--files-out`, `--standalone-files-out`, `--missing-files-out`, `--provenance-out`: output files to write + +## Outputs + +The generator writes five files to the paths passed on the command line: + +- `state-out`: workflow and extraction metadata, including the extracted test inventory. This is the snapshot you can inspect to see which workflows and tests were discovered from the E3SM checkout. +- `files-out`: full-model inputdata URLs confirmed present on the E3SM or CESM server. This is the primary manifest consumed by the inputdata automation and other tooling that needs the full-model file list. +- `standalone-files-out`: standalone EAMxx inputdata URLs. This is used for the separate standalone manifest so those paths can be tracked independently from the full-model list. +- `missing-files-out`: relative paths of input files that could not be confirmed on either the E3SM or CESM server. A `WARNING: missing inputdata` line is also printed to stderr for each entry. Pass `--strict` to fail the run when any missing entries are found. +- `provenance-out`: per-URL provenance showing where each input was discovered. This file is useful for tracing a URL back to the workflow, test, and case directory or source scan that produced it. It is supplemental traceability data, not a required input to the generator. For standalone discoveries, `case_dir` is intentionally empty because there is no CIME case directory. + +## Environment notes + +The generator needs an environment with an E3SM checkout and a working CIME `create_test` script available at `cime/scripts/create_test`. It also expects writable paths for `--test-root` and `--output-root`. + +## Related modules + +- [generate_inputdata_manifest.py](generate_inputdata_manifest.py) +- [build_pr_metadata.py](build_pr_metadata.py) +- [lib/workflow_parser.py](lib/workflow_parser.py) +- [lib/cime_manifest.py](lib/cime_manifest.py) +- [lib/url_normalize.py](lib/url_normalize.py) +- [lib/inputdata_validation.py](lib/inputdata_validation.py) \ No newline at end of file diff --git a/tools/inputdata/__init__.py b/tools/inputdata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/inputdata/build_pr_metadata.py b/tools/inputdata/build_pr_metadata.py new file mode 100644 index 0000000..010d9d5 --- /dev/null +++ b/tools/inputdata/build_pr_metadata.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path + +TRIGGER_FILES = [ + "inputdata/files.txt", + "inputdata/files-standalone.txt", + "inputdata/files-missing.txt", +] + +ADDITIONAL_PR_FILES = [ + "inputdata/e3sm-workflow-state.json", + "inputdata/provenance.json", +] + +PR_FILES = [*TRIGGER_FILES, *ADDITIONAL_PR_FILES] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build PR body content for inputdata manifest updates" + ) + parser.add_argument( + "--output-path", + required=True, + help="Path where the markdown PR body should be written.", + ) + parser.add_argument( + "--state-path", + required=True, + help="Path to inputdata/e3sm-workflow-state.json.", + ) + return parser.parse_args() + + +def _git_numstat(paths: list[str]) -> str: + output = subprocess.check_output( + ["git", "diff", "--numstat", "--", *paths], + text=True, + ) + return output.strip() + + +def _render_body(state: dict, numstat: str) -> str: + tests = state.get("extracted_tests", []) + e3sm_sha = state.get("e3sm_sha", "unknown") + + lines: list[str] = [] + lines.append("Automated inputdata manifest refresh from E3SM workflows.") + lines.append("") + lines.append("Source snapshot") + lines.append(f"- E3SM SHA: {e3sm_sha}") + lines.append(f"- Extracted tests: {len(tests)}") + lines.append("") + lines.append("Changed files") + if numstat: + for row in numstat.splitlines(): + added, removed, path = row.split("\t", maxsplit=2) + lines.append(f"- {path}: +{added} / -{removed} lines") + else: + lines.append("- No managed file changes detected") + lines.append("") + lines.append("PR trigger files") + for path in TRIGGER_FILES: + lines.append(f"- {path}") + lines.append("") + lines.append("Additional files included when a PR is opened") + for path in ADDITIONAL_PR_FILES: + lines.append(f"- {path}") + lines.append("") + lines.append("Review guidance") + lines.append("- Please review the provenance and missing-file updates before merge.") + lines.append( + "- Merging this PR will trigger normal inputdata image build/publish workflows." + ) + + return "\n".join(lines) + "\n" + + +def main() -> int: + args = parse_args() + output_path = Path(args.output_path) + state_path = Path(args.state_path) + + with state_path.open("r", encoding="utf-8") as file: + state = json.load(file) + + numstat = _git_numstat(PR_FILES) + body = _render_body(state, numstat) + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(body, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/inputdata/generate_inputdata_manifest.py b/tools/inputdata/generate_inputdata_manifest.py new file mode 100644 index 0000000..3335b4e --- /dev/null +++ b/tools/inputdata/generate_inputdata_manifest.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from tools.inputdata.lib.cime_manifest import parse_input_data_lists, run_create_test +from tools.inputdata.lib.inputdata_validation import InputDataValidationResult, validate_inputdata_reference +from tools.inputdata.lib.url_normalize import filter_candidate_path, is_public_inputdata_url +from tools.inputdata.lib.workflow_parser import TestRecord, build_state_payload, parse_workflows + + +@dataclass(frozen=True) +class ProvenanceRecord: + workflow: str + test: str + case_dir: str + component_list_file: str + manifest: str + discovery_method: str + relative_path: str + validation_status: str + checked_urls: tuple[str, str] + validation_message: str = "" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate inputdata manifests from E3SM workflows and CIME metadata" + ) + parser.add_argument( + "--e3sm-root", + required=True, + help="Path to the E3SM checkout that provides workflow files and CIME scripts.", + ) + parser.add_argument( + "--test-root", + required=True, + help="Directory where CIME test cases are created during discovery.", + ) + parser.add_argument( + "--output-root", + required=True, + help="Directory used for generated artifacts and temporary discovery output.", + ) + parser.add_argument( + "--state-out", + required=True, + help="File path for the generated workflow and extraction state JSON.", + ) + parser.add_argument( + "--files-out", + required=True, + help="File path for the generated full-model inputdata manifest.", + ) + parser.add_argument( + "--standalone-files-out", + required=True, + help="File path for the generated standalone EAMxx inputdata manifest.", + ) + parser.add_argument( + "--provenance-out", + required=True, + help="File path for the generated provenance JSON.", + ) + parser.add_argument( + "--missing-files-out", + required=True, + help="File path for a list of input files that could not be found on either server.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Fail if workflow parsing or standalone discovery emits warnings.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print progress messages while discovering tests and input files.", + ) + return parser.parse_args() + + +def _write_manifest(urls: list[str], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + text = "\n".join(urls) + if text: + text += "\n" + path.write_text(text, encoding="utf-8") + + +def _write_json(payload: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _log(message: str, verbose: bool) -> None: + if verbose: + print(message) + + +def _collect_standalone_candidate_paths(e3sm_root: Path) -> list[tuple[str, str]]: + """Return (source_path, candidate_input_path) pairs from static scans.""" + candidates: set[tuple[str, str]] = set() + + static_globs = [ + "components/eamxx/scripts/**/*", + "components/eamxx/**/*.yaml", + "components/eamxx/**/*.yml", + ".github/actions/test-all-eamxx/**/*", + ] + + path_pattern = re.compile(r"(?P(?:\$\{?DIN_LOC_ROOT\}?/)?[A-Za-z0-9_./+-]+\.(?:nc|txt|xml|yaml|yml|json|dat|h5))") + + for pattern in static_globs: + for file_path in sorted(e3sm_root.glob(pattern)): + if not file_path.is_file(): + continue + rel_source = file_path.relative_to(e3sm_root).as_posix() + try: + content = file_path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + + for line in content.splitlines(): + if "check-input" not in line and "DIN_LOC_ROOT" not in line and "inputdata" not in line: + continue + for match in path_pattern.finditer(line): + raw = match.group("path") + candidates.add((rel_source, raw)) + + return sorted(candidates) + + +def _run_standalone_config_mode(e3sm_root: Path, output_root: Path) -> tuple[bool, list[str]]: + """Best-effort config-only run for standalone EAMxx discovery.""" + script = e3sm_root / "components" / "eamxx" / "scripts" / "test-all-eamxx" + if not script.exists(): + return False, [f"Missing standalone script: {script}"] + + output_root.mkdir(parents=True, exist_ok=True) + command = [str(script), "--help"] + try: + help_proc = subprocess.run( + command, + cwd=str(e3sm_root), + text=True, + capture_output=True, + ) + except OSError as exc: + return False, [f"Failed to execute standalone script: {exc}"] + + supported_flags = help_proc.stdout + "\n" + help_proc.stderr + run_command = [str(script)] + if "--configure-only" in supported_flags: + run_command.append("--configure-only") + elif "--config-only" in supported_flags: + run_command.append("--config-only") + elif "--dry-run" in supported_flags: + run_command.append("--dry-run") + + if "--work-dir" in supported_flags: + run_command.extend(["--work-dir", str(output_root / "standalone-work")]) + + proc = subprocess.run( + run_command, + cwd=str(e3sm_root), + text=True, + capture_output=True, + ) + + messages: list[str] = [] + if proc.returncode != 0: + tail = (proc.stderr or proc.stdout or "").strip().splitlines() + messages.append("standalone config mode failed") + messages.extend(tail[-10:]) + return False, messages + + return True, messages + + +def _scan_option_b_artifacts(e3sm_root: Path, output_root: Path) -> list[tuple[str, str]]: + candidates: set[tuple[str, str]] = set() + search_roots = [ + output_root, + e3sm_root / "components" / "eamxx" / "ctest-build", + e3sm_root / "components" / "eamxx" / "build", + ] + path_pattern = re.compile(r"(?P(?:\$\{?DIN_LOC_ROOT\}?/)?[A-Za-z0-9_./+-]+\.(?:nc|txt|xml|yaml|yml|json|dat|h5))") + + for root in search_roots: + if not root.exists(): + continue + for file_path in sorted(root.rglob("*")): + if not file_path.is_file(): + continue + if file_path.suffix.lower() not in { + ".txt", + ".cmake", + ".cfg", + ".yaml", + ".yml", + ".log", + ".sh", + ".json", + }: + continue + try: + content = file_path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for line in content.splitlines(): + if "DIN_LOC_ROOT" not in line and "inputdata" not in line: + continue + for match in path_pattern.finditer(line): + raw = match.group("path") + candidates.add((file_path.as_posix(), raw)) + + return sorted(candidates) + + +def _standalone_from_workflow_records( + e3sm_root: Path, + records: list[TestRecord], + din_loc_root: Path, + output_root: Path, + warnings: list[str], + missing_provenance: list[dict[str, object]], +) -> tuple[list[str], dict[str, list[ProvenanceRecord]]]: + urls: set[str] = set() + provenance: dict[str, list[ProvenanceRecord]] = {} + standalone_tests = records or [ + TestRecord(workflow="components/eamxx", kind="standalone_scan", name="standalone-static") + ] + + static_candidates = _collect_standalone_candidate_paths(e3sm_root) + option_b_ok, option_b_messages = _run_standalone_config_mode(e3sm_root, output_root) + if not option_b_ok: + warnings.extend(option_b_messages) + + option_b_candidates = _scan_option_b_artifacts(e3sm_root, output_root) if option_b_ok else [] + all_candidates = static_candidates + option_b_candidates + if not all_candidates: + return [], {} + + for source_path, raw_path in all_candidates: + validation = _normalize_and_validate(raw_path, din_loc_root) + if not validation: + continue + method = "standalone-static" + if source_path.startswith(output_root.as_posix()) or "/ctest-build/" in source_path: + method = "standalone-config" + + for record in standalone_tests: + provenance_record = ProvenanceRecord( + workflow=record.workflow, + test=record.name, + case_dir="", + component_list_file=source_path, + manifest="files-standalone.txt", + discovery_method=method, + relative_path=validation.relative_path, + validation_status=validation.status, + checked_urls=validation.checked_urls, + validation_message=validation.message, + ) + if validation.status == "missing": + missing_provenance.append( + { + "workflow": provenance_record.workflow, + "test": provenance_record.test, + "case_dir": provenance_record.case_dir, + "component_list_file": provenance_record.component_list_file, + "manifest": provenance_record.manifest, + "discovery_method": provenance_record.discovery_method, + "relative_path": provenance_record.relative_path, + "checked_urls": list(provenance_record.checked_urls), + "validation_status": provenance_record.validation_status, + "validation_message": provenance_record.validation_message, + } + ) + continue + + url = validation.resolved_url or validation.checked_urls[0] + urls.add(url) + provenance.setdefault(url, []).append(provenance_record) + + return sorted(urls), provenance + + +def _normalize_and_validate( + raw_value: str, + din_loc_root: Path, +) -> Optional[InputDataValidationResult]: + if not filter_candidate_path(raw_value): + return None + + validation_result = validate_inputdata_reference(raw_value, din_loc_root) + if not validation_result: + return None + + if not validation_result.resolved_url: + return validation_result + + if not is_public_inputdata_url(validation_result.resolved_url): + return None + + return validation_result + + +def _provenance_payload(data: dict[str, list[ProvenanceRecord]]) -> dict[str, list[dict]]: + payload: dict[str, list[dict]] = {} + for url in sorted(data): + entries = sorted( + data[url], + key=lambda rec: ( + rec.workflow, + rec.test, + rec.component_list_file, + rec.case_dir, + rec.manifest, + rec.discovery_method, + rec.validation_status, + rec.relative_path, + ), + ) + payload[url] = [ + { + "workflow": rec.workflow, + "test": rec.test, + "case_dir": rec.case_dir, + "component_list_file": rec.component_list_file, + "manifest": rec.manifest, + "discovery_method": rec.discovery_method, + "relative_path": rec.relative_path, + "validation_status": rec.validation_status, + "checked_urls": list(rec.checked_urls), + "validation_message": rec.validation_message, + } + for rec in entries + ] + return payload + + +def main() -> int: + args = parse_args() + + e3sm_root = Path(args.e3sm_root).resolve() + test_root = Path(args.test_root).resolve() + output_root = Path(args.output_root).resolve() + din_loc_root = (Path.home() / "e3sm-inputdata-empty").resolve() + din_loc_root.mkdir(parents=True, exist_ok=True) + + if not e3sm_root.exists(): + print(f"ERROR: --e3sm-root does not exist: {e3sm_root}", file=sys.stderr) + return 2 + + parse_result = parse_workflows(e3sm_root) + if not parse_result.full_model_tests: + print("ERROR: no full-model tests extracted from workflows", file=sys.stderr) + return 3 + + state = build_state_payload(parse_result) + full_urls: set[str] = set() + full_provenance: dict[str, list[ProvenanceRecord]] = {} + missing_provenance: list[dict[str, object]] = [] + + success_count = 0 + failure_count = 0 + + for test_record in parse_result.full_model_tests: + _log(f"Running create_test for {test_record.name}", args.verbose) + result = run_create_test( + e3sm_root=e3sm_root, + test_name=test_record.name, + test_root=test_root, + output_root=output_root, + din_loc_root=din_loc_root, + workflow=test_record.workflow, + ) + + if not result.success: + failure_count += 1 + print( + "WARNING: create_test failed " + f"test={test_record.name} " + f"started={result.started_at_utc} " + f"finished={result.finished_at_utc} " + f"error={result.error}", + file=sys.stderr, + ) + continue + + success_count += 1 + + for case_dir_value in result.case_dirs: + case_dir = Path(case_dir_value) + entries = parse_input_data_lists(case_dir) + for entry in entries: + validation = _normalize_and_validate(entry.raw_value, din_loc_root) + if not validation: + continue + + provenance_record = ProvenanceRecord( + workflow=test_record.workflow, + test=test_record.name, + case_dir=test_record.name, + component_list_file=entry.component_list_file, + manifest="files.txt", + discovery_method="cime-buildconf", + relative_path=validation.relative_path, + validation_status=validation.status, + checked_urls=validation.checked_urls, + validation_message=validation.message, + ) + + if validation.status == "missing": + missing_provenance.append( + { + "workflow": provenance_record.workflow, + "test": provenance_record.test, + "case_dir": provenance_record.case_dir, + "component_list_file": provenance_record.component_list_file, + "manifest": provenance_record.manifest, + "discovery_method": provenance_record.discovery_method, + "relative_path": provenance_record.relative_path, + "checked_urls": list(provenance_record.checked_urls), + "validation_status": provenance_record.validation_status, + "validation_message": provenance_record.validation_message, + } + ) + continue + + url = validation.resolved_url or validation.checked_urls[0] + full_urls.add(url) + full_provenance.setdefault(url, []).append(provenance_record) + + if success_count == 0: + print("ERROR: CIME case generation failed for all tests", file=sys.stderr) + return 4 + + if not full_urls: + print("ERROR: resulting full-model manifest would be empty", file=sys.stderr) + return 5 + + standalone_warnings: list[str] = [] + standalone_urls, standalone_provenance = _standalone_from_workflow_records( + e3sm_root=e3sm_root, + records=parse_result.standalone_tests, + din_loc_root=din_loc_root, + output_root=output_root, + warnings=standalone_warnings, + missing_provenance=missing_provenance, + ) + + combined_provenance = dict(full_provenance) + for url, entries in standalone_provenance.items(): + combined_provenance.setdefault(url, []).extend(entries) + + files_out = Path(args.files_out) + standalone_files_out = Path(args.standalone_files_out) + provenance_out = Path(args.provenance_out) + missing_files_out = Path(args.missing_files_out) + state_out = Path(args.state_out) + + missing_relative_paths = sorted( + {entry["relative_path"] for entry in missing_provenance} + ) + + provenance_payload = _provenance_payload(combined_provenance) + if missing_provenance: + provenance_payload["missing"] = missing_provenance + + _write_manifest(sorted(full_urls), files_out) + _write_manifest(sorted(standalone_urls), standalone_files_out) + _write_manifest(missing_relative_paths, missing_files_out) + _write_json(provenance_payload, provenance_out) + _write_json(state, state_out) + + for entry in sorted(missing_provenance, key=lambda e: e["relative_path"]): + print( + f"WARNING: missing inputdata " + f"path={entry['relative_path']} " + f"checked={entry['checked_urls']}", + file=sys.stderr, + ) + if missing_provenance and args.strict: + print("ERROR: strict mode enabled and missing inputdata entries were found", file=sys.stderr) + return 8 + + if parse_result.warnings: + for warning in parse_result.warnings: + print(f"WARNING: {warning}", file=sys.stderr) + if args.strict: + print("ERROR: strict mode enabled and parser warnings were present", file=sys.stderr) + return 6 + + if standalone_warnings: + for warning in standalone_warnings: + print(f"WARNING: {warning}", file=sys.stderr) + if args.strict: + print("ERROR: strict mode enabled and standalone warnings were present", file=sys.stderr) + return 7 + + print( + "SUMMARY " + f"full_tests={len(parse_result.full_model_tests)} " + f"full_success={success_count} " + f"full_failed={failure_count} " + f"full_urls={len(full_urls)} " + f"standalone_refs={len(parse_result.standalone_tests)} " + f"standalone_urls={len(standalone_urls)} " + f"missing={len(missing_relative_paths)}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/inputdata/lib/__init__.py b/tools/inputdata/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/inputdata/lib/cime_manifest.py b/tools/inputdata/lib/cime_manifest.py new file mode 100644 index 0000000..5d30574 --- /dev/null +++ b/tools/inputdata/lib/cime_manifest.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import glob +import subprocess +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + + +@dataclass(frozen=True) +class InputListEntry: + case_dir: str + component_list_file: str + raw_value: str + + +@dataclass(frozen=True) +class CaseResult: + test_name: str + workflow: str + success: bool + case_dirs: list[str] + started_at_utc: str + finished_at_utc: str + error: Optional[str] = None + + +def run_create_test( + e3sm_root: Path, + test_name: str, + test_root: Path, + output_root: Path, + din_loc_root: Path, + workflow: str, +) -> CaseResult: + create_test = e3sm_root / "cime" / "scripts" / "create_test" + if not create_test.exists(): + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return CaseResult( + test_name=test_name, + workflow=workflow, + success=False, + case_dirs=[], + started_at_utc=now, + finished_at_utc=now, + error=f"Missing create_test script: {create_test}", + ) + + before = set(discover_case_dirs(test_root)) + started_at_utc = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + command = [ + str(create_test), + test_name, + "--no-build", + "--no-run", + "--namelists-only", + "--generate", + "-b", + "local", + "--allow-baseline-overwrite", + "--baseline-root", + str(output_root), + "--test-root", + str(test_root), + "--output-root", + str(output_root), + "--input-dir", + str(din_loc_root), + "--wait", + ] + + process = subprocess.run( + command, + cwd=str(e3sm_root), + text=True, + capture_output=True, + ) + finished_at_utc = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + after = set(discover_case_dirs(test_root)) + new_case_dirs = sorted(after - before) + case_dirs = new_case_dirs if new_case_dirs else sorted(after) + + if process.returncode != 0: + stderr_tail = (process.stderr or process.stdout or "").strip().splitlines() + tail = "\n".join(stderr_tail[-10:]) if stderr_tail else "create_test failed" + return CaseResult( + test_name=test_name, + workflow=workflow, + success=False, + case_dirs=case_dirs, + started_at_utc=started_at_utc, + finished_at_utc=finished_at_utc, + error=tail, + ) + + return CaseResult( + test_name=test_name, + workflow=workflow, + success=True, + case_dirs=case_dirs, + started_at_utc=started_at_utc, + finished_at_utc=finished_at_utc, + ) + + +def discover_case_dirs(test_root: Path) -> list[str]: + pattern = str(test_root / "**" / "Buildconf") + buildconf_dirs = [Path(path) for path in glob.glob(pattern, recursive=True)] + case_dirs = sorted({str(path.parent) for path in buildconf_dirs}) + return case_dirs + + +def _extract_candidate_from_line(line: str) -> list[str]: + line = line.split("#", 1)[0].strip() + if not line: + return [] + + if "=" in line: + _, rhs = line.split("=", 1) + else: + rhs = line + + raw_tokens = rhs.replace(",", " ").split() + tokens: list[str] = [] + for token in raw_tokens: + cleaned = token.strip().strip('"').strip("'") + if not cleaned: + continue + tokens.append(cleaned) + return tokens + + +def parse_input_data_lists(case_dir: Path) -> list[InputListEntry]: + entries: list[InputListEntry] = [] + buildconf = case_dir / "Buildconf" + for list_file in sorted(buildconf.glob("*.input_data_list")): + with list_file.open("r", encoding="utf-8", errors="replace") as handle: + for line in handle: + for token in _extract_candidate_from_line(line): + entries.append( + InputListEntry( + case_dir=str(case_dir), + component_list_file=list_file.name, + raw_value=token, + ) + ) + return entries diff --git a/tools/inputdata/lib/inputdata_validation.py b/tools/inputdata/lib/inputdata_validation.py new file mode 100644 index 0000000..c342292 --- /dev/null +++ b/tools/inputdata/lib/inputdata_validation.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import ssl +from dataclasses import dataclass +from typing import Optional +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from tools.inputdata.lib.url_normalize import normalize_to_din_relative, to_public_inputdata_url + +CESM_INPUTDATA_PREFIX = "https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata" + + +@dataclass(frozen=True) +class InputDataValidationResult: + relative_path: str + checked_urls: tuple[str, str] + resolved_url: Optional[str] + status: str + message: str = "" + + +@dataclass(frozen=True) +class UrlProbeResult: + exists: bool + verified: bool + message: str = "" + + +def _probe_url(url: str) -> UrlProbeResult: + request = Request( + url, + method="HEAD", + headers={"User-Agent": "Mozilla/5.0", "Accept-Encoding": "identity"}, + ) + context = ssl._create_unverified_context() + try: + with urlopen(request, timeout=20, context=context) as response: + status = getattr(response, "status", 200) + if 200 <= status < 400: + return UrlProbeResult(exists=True, verified=True) + return UrlProbeResult(exists=False, verified=False, message=f"unexpected status {status}") + except HTTPError as exc: + if exc.code in {404, 410}: + return UrlProbeResult(exists=False, verified=True, message=f"http {exc.code}") + return UrlProbeResult(exists=False, verified=False, message=f"http {exc.code}") + except (URLError, OSError, TimeoutError) as exc: + return UrlProbeResult(exists=False, verified=False, message=str(exc)) + + +def validate_inputdata_reference(path_value: str, din_loc_root) -> Optional[InputDataValidationResult]: + relative_path = normalize_to_din_relative(path_value, din_loc_root) + if not relative_path: + return None + + e3sm_url = to_public_inputdata_url(relative_path) + cesm_url = f"{CESM_INPUTDATA_PREFIX}/{relative_path.strip().lstrip('/')}" + + e3sm_probe = _probe_url(e3sm_url) + if e3sm_probe.exists: + return InputDataValidationResult( + relative_path=relative_path, + checked_urls=(e3sm_url, cesm_url), + resolved_url=e3sm_url, + status="e3sm", + ) + + cesm_probe = _probe_url(cesm_url) + if cesm_probe.exists: + return InputDataValidationResult( + relative_path=relative_path, + checked_urls=(e3sm_url, cesm_url), + resolved_url=cesm_url, + status="cesm", + ) + + if e3sm_probe.verified and cesm_probe.verified: + return InputDataValidationResult( + relative_path=relative_path, + checked_urls=(e3sm_url, cesm_url), + resolved_url=None, + status="missing", + message="not found on E3SM or CESM", + ) + + message_parts = [] + if e3sm_probe.message: + message_parts.append(f"e3sm: {e3sm_probe.message}") + if cesm_probe.message: + message_parts.append(f"cesm: {cesm_probe.message}") + message = "; ".join(message_parts) if message_parts else "validation unavailable" + + return InputDataValidationResult( + relative_path=relative_path, + checked_urls=(e3sm_url, cesm_url), + resolved_url=e3sm_url, + status="unverified", + message=message, + ) diff --git a/tools/inputdata/lib/url_normalize.py b/tools/inputdata/lib/url_normalize.py new file mode 100644 index 0000000..4c940f3 --- /dev/null +++ b/tools/inputdata/lib/url_normalize.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +PUBLIC_INPUTDATA_PREFIX = "https://web.lcrc.anl.gov/public/e3sm/inputdata" + + +def filter_candidate_path(value: str) -> bool: + """Return True when a path candidate looks like model inputdata.""" + token = value.strip().strip('"').strip("'") + if not token: + return False + + lowered = token.lower() + if lowered in {"unset", "none", "null", "na", "n/a"}: + return False + + blocked_fragments = ( + "baseline", + "rest", + "restart", + "history", + "output", + "archive", + ) + if any(fragment in lowered for fragment in blocked_fragments): + return False + + if token.endswith("/"): + return False + + # Inputdata files are file-like paths and almost always include separators. + if "/" not in token and "\\" not in token: + return False + + return True + + +def normalize_to_din_relative(path_value: str, din_loc_root: Path) -> Optional[str]: + """Normalize path-like text to a DIN_LOC_ROOT-relative unix-style path.""" + value = path_value.strip().strip('"').strip("'") + value = value.replace("\\", "/") + if value.startswith("$DIN_LOC_ROOT/"): + rel = value[len("$DIN_LOC_ROOT/") :] + return rel.strip("/") + + if value.startswith("${DIN_LOC_ROOT}/"): + rel = value[len("${DIN_LOC_ROOT}/") :] + return rel.strip("/") + + din_root_posix = din_loc_root.as_posix().rstrip("/") + "/" + if value.startswith(din_root_posix): + rel = value[len(din_root_posix) :] + return rel.strip("/") + + # Allow already-relative references such as atm/cam/inic/file.nc. + if not value.startswith("/"): + return value.strip("/") + + return None + + +def to_public_inputdata_url(relative_path: str) -> str: + rel = relative_path.strip().lstrip("/") + return f"{PUBLIC_INPUTDATA_PREFIX}/{rel}" + + +def is_public_inputdata_url(url: str) -> bool: + return url.startswith(PUBLIC_INPUTDATA_PREFIX + "/") diff --git a/tools/inputdata/lib/workflow_parser.py b/tools/inputdata/lib/workflow_parser.py new file mode 100644 index 0000000..b8869e9 --- /dev/null +++ b/tools/inputdata/lib/workflow_parser.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +CREATE_TEST_PATTERN = re.compile( + r"(?:^|\s)(?:\./)?cime/scripts/create_test\s+(?P[A-Za-z0-9_\-.+]+)|" + r"(?:^|\s)create_test\s+(?P[A-Za-z0-9_\-.+]+)" +) +MATRIX_FULL_NAME_PATTERN = re.compile( + r"full_name\s*:\s*[\"']?(?P[A-Za-z0-9_\-.+]+)[\"']?" +) +STANDALONE_PATTERN = re.compile(r"test-all-eamxx") + + +@dataclass(frozen=True) +class TestRecord: + workflow: str + kind: str + name: str + + +@dataclass(frozen=True) +class WorkflowParseResult: + e3sm_sha: str + workflow_hashes: dict[str, str] + full_model_tests: list[TestRecord] + standalone_tests: list[TestRecord] + warnings: list[str] + + + +def collect_workflow_files(e3sm_root: Path) -> list[Path]: + workflows_dir = e3sm_root / ".github" / "workflows" + files = sorted(workflows_dir.glob("*.yml")) + sorted(workflows_dir.glob("*.yaml")) + return sorted(set(files)) + + +def _hash_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _get_e3sm_sha(e3sm_root: Path) -> str: + try: + output = subprocess.check_output( + ["git", "-C", str(e3sm_root), "rev-parse", "HEAD"], + text=True, + ) + return output.strip() + except Exception: + return "unknown" + + +def _parse_create_test_lines(workflow_rel: str, content: str) -> Iterable[TestRecord]: + seen: set[str] = set() + for line in content.splitlines(): + match = CREATE_TEST_PATTERN.search(line) + if not match: + continue + test_name = match.group("test") or match.group("test2") + if not test_name: + continue + test_name = test_name.strip().strip('"').strip("'") + if test_name not in seen: + seen.add(test_name) + yield TestRecord(workflow=workflow_rel, kind="create_test", name=test_name) + + + +def _parse_matrix_full_names(workflow_rel: str, content: str) -> Iterable[TestRecord]: + seen: set[str] = set() + for line in content.splitlines(): + match = MATRIX_FULL_NAME_PATTERN.search(line) + if not match: + continue + test_name = match.group("test").strip() + if test_name not in seen: + seen.add(test_name) + yield TestRecord(workflow=workflow_rel, kind="matrix_full_name", name=test_name) + + + +def _parse_standalone(workflow_rel: str, content: str) -> Iterable[TestRecord]: + lines = content.splitlines() + for idx, line in enumerate(lines): + if not STANDALONE_PATTERN.search(line): + continue + name = f"standalone_ref_line_{idx + 1}" + yield TestRecord(workflow=workflow_rel, kind="standalone_ref", name=name) + + + +def parse_workflows(e3sm_root: Path) -> WorkflowParseResult: + workflow_files = collect_workflow_files(e3sm_root) + if not workflow_files: + raise FileNotFoundError("No workflow YAML files found under E3SM/.github/workflows") + + workflow_hashes: dict[str, str] = {} + full_records: set[TestRecord] = set() + standalone_records: set[TestRecord] = set() + warnings: list[str] = [] + + for path in workflow_files: + relative = path.relative_to(e3sm_root).as_posix() + workflow_hashes[relative] = _hash_file(path) + content = path.read_text(encoding="utf-8", errors="replace") + + full_before = len(full_records) + for record in _parse_create_test_lines(relative, content): + full_records.add(record) + for record in _parse_matrix_full_names(relative, content): + full_records.add(record) + + for record in _parse_standalone(relative, content): + standalone_records.add(record) + + if len(full_records) == full_before and "create_test" in content: + warnings.append(f"Could not confidently parse create_test syntax in {relative}") + + return WorkflowParseResult( + e3sm_sha=_get_e3sm_sha(e3sm_root), + workflow_hashes=dict(sorted(workflow_hashes.items())), + full_model_tests=sorted(full_records, key=lambda r: (r.workflow, r.kind, r.name)), + standalone_tests=sorted(standalone_records, key=lambda r: (r.workflow, r.kind, r.name)), + warnings=sorted(set(warnings)), + ) + + +def build_state_payload(parse_result: WorkflowParseResult, e3sm_ref: str = "master") -> dict: + extracted_tests = [ + {"workflow": rec.workflow, "kind": rec.kind, "name": rec.name} + for rec in parse_result.full_model_tests + parse_result.standalone_tests + ] + extracted_tests = sorted( + extracted_tests, + key=lambda r: (r["workflow"], r["kind"], r["name"]), + ) + + encoded = json.dumps(extracted_tests, sort_keys=True, separators=(",", ":")) + extracted_hash = hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + return { + "e3sm_ref": e3sm_ref, + "e3sm_sha": parse_result.e3sm_sha, + "workflow_files_sha256": parse_result.workflow_hashes, + "extracted_tests_sha256": extracted_hash, + "extracted_tests": extracted_tests, + }