diff --git a/.github/workflows/forge-existing-release-reconcile.yml b/.github/workflows/forge-existing-release-reconcile.yml new file mode 100644 index 0000000..cc9c3de --- /dev/null +++ b/.github/workflows/forge-existing-release-reconcile.yml @@ -0,0 +1,502 @@ +name: Forge existing release reconciliation + +on: + workflow_dispatch: + inputs: + version: + description: Existing Forge version already published to PyPI + required: true + type: string + product_source_sha: + description: Immutable protected source that produced the existing artifacts + required: true + type: string + original_run_id: + description: Failed original Forge production release run containing qualified artifacts + required: true + type: string + wheel_sha256: + description: Qualified wheel SHA-256 without prefix + required: true + type: string + sdist_sha256: + description: Qualified source-distribution SHA-256 without prefix + required: true + type: string + +permissions: + contents: read + +concurrency: + group: forge-production-release + cancel-in-progress: false + +jobs: + # Draft releases require a write-capable token to be read. This job never + # downloads or executes package code and performs no release mutation. + snapshot-draft-release: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5 + with: + fetch-depth: 0 + persist-credentials: false + - name: Bind the protected controller and capture every release asset + env: + VERSION: ${{ inputs.version }} + PRODUCT_SOURCE_SHA: ${{ inputs.product_source_sha }} + ORIGINAL_RUN_ID: ${{ inputs.original_run_id }} + WHEEL_SHA256: ${{ inputs.wheel_sha256 }} + SDIST_SHA256: ${{ inputs.sdist_sha256 }} + run: | + set -euo pipefail + git fetch --no-tags origin main + test "$GITHUB_SHA" = "$(git rev-parse HEAD)" + test "$GITHUB_SHA" = "$(git rev-parse origin/main)" + [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] + [[ "$PRODUCT_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$ORIGINAL_RUN_ID" =~ ^[1-9][0-9]*$ ]] + [[ "$WHEEL_SHA256" =~ ^[0-9a-f]{64}$ ]] + [[ "$SDIST_SHA256" =~ ^[0-9a-f]{64}$ ]] + git merge-base --is-ancestor "$PRODUCT_SOURCE_SHA" origin/main + tag="forge-v$VERSION" + forge_gh_token="${{ github.token }}" + GH_TOKEN="$forge_gh_token" gh release view "$tag" --repo "$GITHUB_REPOSITORY" \ + --json apiUrl,assets,createdAt,databaseId,id,isDraft,isImmutable,isPrerelease,tagName,targetCommitish,url \ + > release-inventory.json + python3 scripts/reconcile_existing_release.py inventory \ + --release-json release-inventory.json --version "$VERSION" \ + --source-revision "$PRODUCT_SOURCE_SHA" > release-inventory-validated.json + mkdir release-assets + GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --dir release-assets + python3 scripts/reconcile_existing_release.py validate-assets \ + --release-json release-inventory.json --asset-root release-assets \ + --version "$VERSION" --source-revision "$PRODUCT_SOURCE_SHA" > release-assets-validated.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: forge-release-snapshot-${{ inputs.version }}-${{ github.sha }} + path: | + release-inventory.json + release-assets/ + if-no-files-found: error + + validate-existing-release: + needs: snapshot-draft-release + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: forge-release-snapshot-${{ inputs.version }}-${{ github.sha }} + path: release-snapshot + - name: Recover the original operation and qualify fresh PyPI readback + env: + VERSION: ${{ inputs.version }} + PRODUCT_SOURCE_SHA: ${{ inputs.product_source_sha }} + ORIGINAL_RUN_ID: ${{ inputs.original_run_id }} + WHEEL_SHA256: ${{ inputs.wheel_sha256 }} + SDIST_SHA256: ${{ inputs.sdist_sha256 }} + run: | + set -euo pipefail + operation="forge-release-$VERSION-$PRODUCT_SOURCE_SHA" + artifact="forge-autonomy-$VERSION-$PRODUCT_SOURCE_SHA" + qualified="forge-release-qualified-$VERSION-$PRODUCT_SOURCE_SHA.json" + forge_gh_token="${{ github.token }}" + GH_TOKEN="$forge_gh_token" gh run view "$ORIGINAL_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --json databaseId,status,conclusion,headSha,event,workflowName,jobs > original-run.json + mkdir original-action + GH_TOKEN="$forge_gh_token" gh run download "$ORIGINAL_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --name "$artifact" --dir original-action + test -f "original-action/release-evidence/operations/$operation.json" + test -f original-action/dist/SHA256SUMS + wheel="forge_autonomy-$VERSION-py3-none-any.whl" + sdist="forge_autonomy-$VERSION.tar.gz" + grep -Fxq "$WHEEL_SHA256 dist/$wheel" original-action/dist/SHA256SUMS + grep -Fxq "$SDIST_SHA256 dist/$sdist" original-action/dist/SHA256SUMS + cmp "original-action/release-evidence/operations/$operation.json" \ + "release-snapshot/release-assets/$qualified" + python3 scripts/pypi_distribution_readback.py \ + --version "$VERSION" --hashes original-action/dist/SHA256SUMS \ + --output pypi.json --cache-token "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + --attempts 24 --interval-seconds 15 --timeout-seconds 10 + mkdir registry-readback + for filename in "$wheel" "$sdist"; do + artifact_url="$(python3 - "$filename" <<'PY' + import json + import sys + print(next(item["url"] for item in json.load(open("pypi.json", encoding="utf-8"))["urls"] if item["filename"] == sys.argv[1])) + PY + )" + curl --fail --silent --show-error --location "$artifact_url" -o "registry-readback/$filename" + done + python3 scripts/reconcile_existing_release.py validate \ + --run-json original-run.json \ + --qualified-action "original-action/release-evidence/operations/$operation.json" \ + --qualified-release "release-snapshot/release-assets/$qualified" \ + --hashes original-action/dist/SHA256SUMS \ + --wheel "registry-readback/$wheel" --sdist "registry-readback/$sdist" \ + --version "$VERSION" --source-revision "$PRODUCT_SOURCE_SHA" \ + --original-run-id "$ORIGINAL_RUN_ID" --controller-source "$GITHUB_SHA" \ + --reconciliation-run-id "$GITHUB_RUN_ID" \ + --release-json release-snapshot/release-inventory.json > publication.json + cmp "original-action/dist/$wheel" "registry-readback/$wheel" + cmp "original-action/dist/$sdist" "registry-readback/$sdist" + mkdir validation-bundle + cp -R original-action/release-evidence validation-bundle/ + cp publication.json release-snapshot/release-inventory.json validation-bundle/ + cp -R release-snapshot/release-assets validation-bundle/ + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: forge-release-validation-${{ inputs.version }}-${{ github.sha }} + path: validation-bundle/ + if-no-files-found: error + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: forge-release-smoke-${{ inputs.version }}-${{ github.sha }} + path: registry-readback/forge_autonomy-*.whl + if-no-files-found: error + + smoke-existing-release: + needs: validate-existing-release + runs-on: ubuntu-latest + permissions: + actions: read + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: forge-release-smoke-${{ inputs.version }}-${{ github.sha }} + path: registry-wheel + - name: Execute the qualified wheel without credentials or a privileged handoff + env: + VERSION: ${{ inputs.version }} + WHEEL_SHA256: ${{ inputs.wheel_sha256 }} + run: | + set -euo pipefail + wheel="registry-wheel/forge_autonomy-$VERSION-py3-none-any.whl" + test "$(sha256sum "$wheel" | cut -d' ' -f1)" = "$WHEEL_SHA256" + python3 -m venv "$RUNNER_TEMP/forge-registry-wheel" + "$RUNNER_TEMP/forge-registry-wheel/bin/pip" install --no-index --no-deps "$wheel" + mkdir "$RUNNER_TEMP/forge-smoke-home" + cd "$RUNNER_TEMP" + env -i HOME="$RUNNER_TEMP/forge-smoke-home" TMPDIR="$RUNNER_TEMP" \ + PATH="$RUNNER_TEMP/forge-registry-wheel/bin:/usr/bin:/bin" VERSION="$VERSION" \ + PYTHONNOUSERSITE=1 PYTHONSAFEPATH=1 \ + "$RUNNER_TEMP/forge-registry-wheel/bin/python" -I -c \ + 'import forge,pathlib; from importlib.metadata import version; from forge.qualification.installed_smoke import run; assert version("forge-autonomy") == __import__("os").environ["VERSION"]; assert "/site-packages/forge/" in str(pathlib.Path(forge.__file__).resolve()); run()' + test "$(env -i HOME="$RUNNER_TEMP/forge-smoke-home" TMPDIR="$RUNNER_TEMP" \ + PATH="$RUNNER_TEMP/forge-registry-wheel/bin:/usr/bin:/bin" \ + "$RUNNER_TEMP/forge-registry-wheel/bin/forge" --version)" = "$VERSION" + + reconcile-existing-release: + needs: [validate-existing-release, smoke-existing-release] + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: forge-release-validation-${{ inputs.version }}-${{ github.sha }} + path: validation-bundle + - name: Resume the exact operation, publish the draft, clean up, and verify + env: + VERSION: ${{ inputs.version }} + PRODUCT_SOURCE_SHA: ${{ inputs.product_source_sha }} + ORIGINAL_RUN_ID: ${{ inputs.original_run_id }} + WHEEL_SHA256: ${{ inputs.wheel_sha256 }} + SDIST_SHA256: ${{ inputs.sdist_sha256 }} + run: | + set -euo pipefail + operation="forge-release-$VERSION-$PRODUCT_SOURCE_SHA" + tag="forge-v$VERSION" + qualified="forge-release-qualified-$VERSION-$PRODUCT_SOURCE_SHA.json" + published="forge-release-published-$VERSION-$PRODUCT_SOURCE_SHA.json" + pending="forge-cleanup-pending-$VERSION-$PRODUCT_SOURCE_SHA.json" + complete="forge-release-complete-$VERSION-$PRODUCT_SOURCE_SHA.json" + evidence_root=validation-bundle/release-evidence + forge_gh_token="${{ github.token }}" + git fetch --no-tags origin main + test "$GITHUB_SHA" = "$(git rev-parse origin/main)" + git merge-base --is-ancestor "$PRODUCT_SOURCE_SHA" origin/main + + snapshot_release() { + destination="$1" + assets="$2" + GH_TOKEN="$forge_gh_token" gh release view "$tag" --repo "$GITHUB_REPOSITORY" \ + --json apiUrl,assets,createdAt,databaseId,id,isDraft,isImmutable,isPrerelease,tagName,targetCommitish,url \ + > "$destination" + python3 scripts/reconcile_existing_release.py same-release \ + --expected-release-json validation-bundle/release-inventory.json \ + --current-release-json "$destination" --version "$VERSION" \ + --source-revision "$PRODUCT_SOURCE_SHA" > /dev/null + mkdir "$assets" + GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" --dir "$assets" + python3 scripts/reconcile_existing_release.py validate-assets \ + --release-json "$destination" --asset-root "$assets" \ + --version "$VERSION" --source-revision "$PRODUCT_SOURCE_SHA" > /dev/null + } + verify_tag_state() { + release_json="$1" + draft="$(python3 -c 'import json,sys; print(str(json.load(open(sys.argv[1]))["isDraft"]).lower())' "$release_json")" + remote_tag="$(git ls-remote origin "refs/tags/$tag")" + if test "$draft" = true; then + test -z "$remote_tag" || { echo 'Draft release unexpectedly has a remote tag.' >&2; exit 1; } + else + test -n "$remote_tag" || { echo 'Published release lacks its remote tag.' >&2; exit 1; } + git fetch --force origin "refs/tags/$tag:refs/tags/$tag" + test "$(git rev-parse "$tag^{commit}")" = "$PRODUCT_SOURCE_SHA" + fi + } + verify_controller_runs() { + controller_evidence="$1" + while IFS=$'\t' read -r phase run_id controller_source; do + test -n "$phase" && test -n "$run_id" && test -n "$controller_source" + git merge-base --is-ancestor "$controller_source" origin/main + run_json="$(mktemp "$RUNNER_TEMP/forge-controller-run-XXXXXX.json")" + GH_TOKEN="$forge_gh_token" gh run view "$run_id" --repo "$GITHUB_REPOSITORY" \ + --json databaseId,status,conclusion,headSha,event,workflowName > "$run_json" + python3 scripts/reconcile_existing_release.py validate-controller-run \ + --run-json "$run_json" --controller-source "$controller_source" \ + --reconciliation-run-id "$run_id" > /dev/null + rm -f -- "$run_json" + done < <(python3 - "$controller_evidence" <<'PY' + import json + import sys + evidence = json.load(open(sys.argv[1], encoding="utf-8")) + for item in evidence["controller_runs"]: + print(item["phase"], item["run_id"], item["controller_source"], sep="\t") + PY + ) + } + + python3 - "$GITHUB_RUN_ID" "$GITHUB_SHA" > current-controller.json <<'PY' + import json + import sys + print(json.dumps({"controller_runs": [{ + "phase": "current", "run_id": sys.argv[1], "controller_source": sys.argv[2], + }]}, sort_keys=True)) + PY + verify_controller_runs current-controller.json + + snapshot_release current-before.json current-assets-before + for original in validation-bundle/release-assets/*; do + cmp "$original" "current-assets-before/$(basename "$original")" + done + cmp "$evidence_root/operations/$operation.json" "current-assets-before/$qualified" + verify_tag_state current-before.json + + state=QUALIFIED + if test -f "current-assets-before/$published"; then + followup_args=(--published "current-assets-before/$published") + test ! -f "current-assets-before/$pending" || followup_args+=(--pending "current-assets-before/$pending") + test ! -f "current-assets-before/$complete" || followup_args+=(--complete "current-assets-before/$complete") + python3 scripts/reconcile_existing_release.py validate-followup \ + "${followup_args[@]}" --original-run-id "$ORIGINAL_RUN_ID" \ + --release-json current-before.json > followup-state.json + verify_controller_runs followup-state.json + state="$(python3 -c 'import json; print(json.load(open("followup-state.json"))["state"])')" + receipt="$(python3 -c 'import json; print(json.load(open("followup-state.json"))["receipt"])')" + python3 scripts/reconcile_existing_release.py hydrate \ + --evidence-root "$evidence_root" --operation-id "$operation" \ + --receipt "$receipt" > hydrated.json + fi + + if test "$state" = RELEASE_COMPLETE; then + test "$(python3 -c 'import json; print(str(json.load(open("current-before.json"))["isDraft"]).lower())')" = false + verify_tag_state current-before.json + exit 0 + fi + + if test "$state" = QUALIFIED; then + test "$(python3 -c 'import json; print(str(json.load(open("current-before.json"))["isDraft"]).lower())')" = true + test "$(python3 -c 'import json; print(str(json.load(open("current-before.json"))["isImmutable"]).lower())')" = false + python3 scripts/reconcile_existing_release.py validate-publication \ + --qualified "$evidence_root/operations/$operation.json" \ + --publication validation-bundle/publication.json \ + --original-run-id "$ORIGINAL_RUN_ID" --controller-source "$GITHUB_SHA" \ + --reconciliation-run-id "$GITHUB_RUN_ID" \ + --release-json current-before.json > new-publication-validated.json + verify_controller_runs new-publication-validated.json + python3 scripts/release_operation.py \ + --mark-published --evidence-root "$evidence_root" --operation-id "$operation" \ + --version "$VERSION" --policy-revision forge-bootstrap-release-cadence-v2 \ + --source-revision "$PRODUCT_SOURCE_SHA" \ + --wheel-digest "sha256:$WHEEL_SHA256" --sdist-digest "sha256:$SDIST_SHA256" \ + --evidence-file validation-bundle/publication.json > published-operation.json + cp "$evidence_root/operations/$operation.json" "$published" + published_readback="$(mktemp -d "$RUNNER_TEMP/forge-published-readback-XXXXXX")" + if GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --pattern "$published" --dir "$published_readback"; then + cmp "$published" "$published_readback/$published" + else + GH_TOKEN="$forge_gh_token" gh release upload "$tag" "$published" --repo "$GITHUB_REPOSITORY" + GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --pattern "$published" --dir "$published_readback" + cmp "$published" "$published_readback/$published" + fi + rm -rf -- "$published_readback" + test ! -e "$published_readback" && test ! -L "$published_readback" + state=PUBLISHED + fi + + snapshot_release prepublish.json prepublish-assets + followup_args=(--published "prepublish-assets/$published") + test ! -f "prepublish-assets/$pending" || followup_args+=(--pending "prepublish-assets/$pending") + test ! -f "prepublish-assets/$complete" || followup_args+=(--complete "prepublish-assets/$complete") + python3 scripts/reconcile_existing_release.py validate-followup \ + "${followup_args[@]}" --original-run-id "$ORIGINAL_RUN_ID" \ + --release-json prepublish.json > prepublish-followup.json + verify_controller_runs prepublish-followup.json + prepublish_state="$(python3 -c 'import json; print(json.load(open("prepublish-followup.json"))["state"])')" + if test "$prepublish_state" = RELEASE_COMPLETE; then + test "$(python3 -c 'import json; print(str(json.load(open("prepublish.json"))["isDraft"]).lower())')" = false + verify_tag_state prepublish.json + exit 0 + fi + if test "$prepublish_state" = CLEANUP_PENDING && test "$state" != CLEANUP_PENDING; then + python3 scripts/reconcile_existing_release.py hydrate \ + --evidence-root "$evidence_root" --operation-id "$operation" \ + --receipt "prepublish-assets/$pending" > hydrated.json + fi + state="$prepublish_state" + if test "$(python3 -c 'import json; print(str(json.load(open("prepublish.json"))["isDraft"]).lower())')" = true; then + test "$state" = PUBLISHED + test "$(python3 -c 'import json; print(str(json.load(open("prepublish.json"))["isImmutable"]).lower())')" = false + verify_tag_state prepublish.json + GH_TOKEN="$forge_gh_token" gh release edit "$tag" --repo "$GITHUB_REPOSITORY" --draft=false + fi + + snapshot_release verified-release.json published-assets + test "$(python3 -c 'import json; print(str(json.load(open("verified-release.json"))["isDraft"]).lower())')" = false + verify_tag_state verified-release.json + followup_args=(--published "published-assets/$published") + test ! -f "published-assets/$pending" || followup_args+=(--pending "published-assets/$pending") + test ! -f "published-assets/$complete" || followup_args+=(--complete "published-assets/$complete") + python3 scripts/reconcile_existing_release.py validate-followup \ + "${followup_args[@]}" --original-run-id "$ORIGINAL_RUN_ID" \ + --release-json verified-release.json > verified-followup.json + verify_controller_runs verified-followup.json + verified_state="$(python3 -c 'import json; print(json.load(open("verified-followup.json"))["state"])')" + if test "$verified_state" = RELEASE_COMPLETE; then + exit 0 + fi + if test "$verified_state" = CLEANUP_PENDING && test "$state" != CLEANUP_PENDING; then + python3 scripts/reconcile_existing_release.py hydrate \ + --evidence-root "$evidence_root" --operation-id "$operation" \ + --receipt "published-assets/$pending" > hydrated.json + fi + state="$verified_state" + test "$(python3 -c 'import json; print(str(json.load(open("verified-release.json"))["isImmutable"]).lower())')" = false + + read -r wheel_digest sdist_digest < <(python3 - "$evidence_root/operations/$operation.json" <<'PY' + import json + import sys + operation = json.load(open(sys.argv[1], encoding="utf-8")) + print(operation["artifacts"]["wheel"], operation["artifacts"]["sdist"]) + PY + ) + cleanup_targets=( + current-assets-before prepublish-assets published-assets validation-bundle/release-assets + validation-bundle/publication.json validation-bundle/release-inventory.json + current-controller.json current-before.json prepublish.json followup-state.json hydrated.json + new-publication-validated.json published-operation.json + prepublish-followup.json verified-followup.json "$published" + ) + cleanup_failed=0 + failed_targets=() + for target in "${cleanup_targets[@]}"; do + if test -e "$target" || test -L "$target"; then + if ! rm -rf -- "$target" || test -e "$target" || test -L "$target"; then + cleanup_failed=1 + failed_targets+=("$target") + fi + fi + done + if test "$cleanup_failed" = 1; then + if test "$state" = CLEANUP_PENDING; then + echo 'Existing durable CLEANUP_PENDING receipt retained after repeat cleanup failure.' >&2 + exit 1 + fi + pending_args=() + for failed in "${failed_targets[@]}"; do pending_args+=(--failed-target "$failed"); done + python3 scripts/reconcile_existing_release.py cleanup-pending-evidence \ + --version "$VERSION" --original-run-id "$ORIGINAL_RUN_ID" \ + --controller-source "$GITHUB_SHA" --reconciliation-run-id "$GITHUB_RUN_ID" \ + --source-revision "$PRODUCT_SOURCE_SHA" --release-json verified-release.json \ + "${pending_args[@]}" > cleanup-pending.json + python3 scripts/release_operation.py \ + --mark-cleanup-pending --evidence-root "$evidence_root" --operation-id "$operation" \ + --version "$VERSION" --policy-revision forge-bootstrap-release-cadence-v2 \ + --source-revision "$PRODUCT_SOURCE_SHA" --wheel-digest "$wheel_digest" \ + --sdist-digest "$sdist_digest" --evidence-file cleanup-pending.json > /dev/null + cp "$evidence_root/operations/$operation.json" "$pending" + pending_readback="$(mktemp -d "$RUNNER_TEMP/forge-pending-readback-XXXXXX")" + if ! GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --pattern "$pending" --dir "$pending_readback"; then + GH_TOKEN="$forge_gh_token" gh release upload "$tag" "$pending" --repo "$GITHUB_REPOSITORY" + GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --pattern "$pending" --dir "$pending_readback" + fi + cmp "$pending" "$pending_readback/$pending" + rm -rf -- "$pending_readback" + exit 1 + fi + + python3 scripts/reconcile_existing_release.py completion-evidence \ + --version "$VERSION" --original-run-id "$ORIGINAL_RUN_ID" \ + --controller-source "$GITHUB_SHA" --reconciliation-run-id "$GITHUB_RUN_ID" \ + --source-revision "$PRODUCT_SOURCE_SHA" --release-json verified-release.json \ + > completion.json + python3 scripts/release_operation.py \ + --complete --evidence-root "$evidence_root" --operation-id "$operation" \ + --version "$VERSION" --policy-revision forge-bootstrap-release-cadence-v2 \ + --source-revision "$PRODUCT_SOURCE_SHA" --wheel-digest "$wheel_digest" \ + --sdist-digest "$sdist_digest" --evidence-file completion.json > complete-operation.json + cp "$evidence_root/operations/$operation.json" "$complete" + complete_readback="$(mktemp -d "$RUNNER_TEMP/forge-complete-readback-XXXXXX")" + if ! GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --pattern "$complete" --dir "$complete_readback"; then + GH_TOKEN="$forge_gh_token" gh release upload "$tag" "$complete" --repo "$GITHUB_REPOSITORY" + GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --pattern "$complete" --dir "$complete_readback" + fi + cmp "$complete" "$complete_readback/$complete" + rm -rf -- "$complete_readback" + + GH_TOKEN="$forge_gh_token" gh release view "$tag" --repo "$GITHUB_REPOSITORY" \ + --json apiUrl,assets,createdAt,databaseId,id,isDraft,isImmutable,isPrerelease,tagName,targetCommitish,url \ + > final-release.json + python3 scripts/reconcile_existing_release.py same-release \ + --expected-release-json verified-release.json --current-release-json final-release.json \ + --version "$VERSION" --source-revision "$PRODUCT_SOURCE_SHA" > /dev/null + final_assets="$(mktemp -d "$RUNNER_TEMP/forge-final-assets-XXXXXX")" + GH_TOKEN="$forge_gh_token" gh release download "$tag" --repo "$GITHUB_REPOSITORY" --dir "$final_assets" + python3 scripts/reconcile_existing_release.py validate-assets \ + --release-json final-release.json --asset-root "$final_assets" \ + --version "$VERSION" --source-revision "$PRODUCT_SOURCE_SHA" > /dev/null + final_followup_args=(--published "$final_assets/$published") + test ! -f "$final_assets/$pending" || final_followup_args+=(--pending "$final_assets/$pending") + final_followup_args+=(--complete "$final_assets/$complete") + python3 scripts/reconcile_existing_release.py validate-followup \ + "${final_followup_args[@]}" --original-run-id "$ORIGINAL_RUN_ID" \ + --release-json final-release.json > final-followup.json + verify_controller_runs final-followup.json + test "$(python3 -c 'import json; print(json.load(open("final-followup.json"))["state"])')" = RELEASE_COMPLETE + test "$(python3 -c 'import json; print(str(json.load(open("final-release.json"))["isDraft"]).lower())')" = false + verify_tag_state final-release.json + rm -rf -- "$final_assets" + test ! -e "$final_assets" && test ! -L "$final_assets" diff --git a/scripts/reconcile_existing_release.py b/scripts/reconcile_existing_release.py new file mode 100644 index 0000000..8ff2e8b --- /dev/null +++ b/scripts/reconcile_existing_release.py @@ -0,0 +1,723 @@ +#!/usr/bin/env python3 +"""Validate and resume one already-published Forge release without uploading it again.""" + +from __future__ import annotations + +import argparse +from dataclasses import asdict +from email.parser import Parser +from hashlib import sha256 +import json +from pathlib import Path +import re +import sys +import tarfile +from typing import Mapping +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from pypi_distribution_readback import distribution_filenames, expected_digests_from_sha256sums +from release_operation import ReleaseOperation, ReleaseOperationError, ReleaseOperationStore + + +RECONCILIATION_CONTRACT = "forge-existing-release-reconciliation/v1" +_SHA256 = re.compile(r"sha256:[0-9a-f]{64}\Z") +_REVISION = re.compile(r"[0-9a-f]{40}\Z") +_RUN_ID = re.compile(r"[1-9][0-9]*\Z") +_STATE_RANK = { + "PREPARED": 0, + "QUALIFIED": 1, + "PUBLISHED": 2, + "CLEANUP_PENDING": 3, + "RELEASE_COMPLETE": 4, +} + + +class ExistingReleaseReconciliationError(RuntimeError): + """The supplied historical release evidence cannot be reconciled safely.""" + + +def _load_json(path: Path) -> object: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ExistingReleaseReconciliationError(f"release evidence is unreadable: {path.name}") from error + + +def _digest(path: Path) -> str: + supplied = Path(path) + if supplied.is_symlink(): + raise ExistingReleaseReconciliationError(f"release artifact is unavailable or unsafe: {path.name}") + candidate = supplied.resolve() + if not candidate.is_file(): + raise ExistingReleaseReconciliationError(f"release artifact is unavailable or unsafe: {path.name}") + value = sha256() + with candidate.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return "sha256:" + value.hexdigest() + + +def _validate_original_run(document: object, *, run_id: str, source_revision: str) -> None: + if not isinstance(document, Mapping): + raise ExistingReleaseReconciliationError("original release run evidence is invalid") + expected = { + "databaseId": int(run_id), + "status": "completed", + "conclusion": "failure", + "headSha": source_revision, + "event": "workflow_dispatch", + "workflowName": "Forge production release", + } + for key, value in expected.items(): + if document.get(key) != value: + raise ExistingReleaseReconciliationError(f"original release run {key} does not match") + jobs = document.get("jobs") + if not isinstance(jobs, list) or any(not isinstance(item, Mapping) for item in jobs): + raise ExistingReleaseReconciliationError("original release run jobs are invalid") + conclusions = {str(item.get("name")): item.get("conclusion") for item in jobs} + required = { + "build-and-qualify": "success", + "publish-pypi": "success", + "registry-readback-and-published-evidence": "failure", + "record-release-complete": "skipped", + } + if any(conclusions.get(name) != conclusion for name, conclusion in required.items()): + raise ExistingReleaseReconciliationError("original release run did not fail at the expected post-publication boundary") + + +def _validate_package_metadata(wheel: Path, sdist: Path, *, version: str) -> None: + try: + with zipfile.ZipFile(wheel) as archive: + metadata_names = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")] + if len(metadata_names) != 1: + raise ExistingReleaseReconciliationError("wheel metadata is missing or ambiguous") + wheel_metadata = Parser().parsestr(archive.read(metadata_names[0]).decode("utf-8")) + with tarfile.open(sdist, "r:gz") as archive: + metadata_members = [ + member for member in archive.getmembers() + if member.name.endswith("/PKG-INFO") and member.name.count("/") == 1 + ] + if len(metadata_members) != 1: + raise ExistingReleaseReconciliationError("source-distribution metadata is missing or ambiguous") + extracted = archive.extractfile(metadata_members[0]) + if extracted is None: + raise ExistingReleaseReconciliationError("source-distribution metadata is unavailable") + sdist_metadata = Parser().parsestr(extracted.read().decode("utf-8")) + except (OSError, UnicodeDecodeError, zipfile.BadZipFile, tarfile.TarError) as error: + raise ExistingReleaseReconciliationError("published distribution metadata is unreadable") from error + for label, metadata in (("wheel", wheel_metadata), ("source distribution", sdist_metadata)): + if metadata.get("Name") != "forge-autonomy" or metadata.get("Version") != version: + raise ExistingReleaseReconciliationError(f"{label} package identity does not match") + + +def validate_reconciliation( + *, + run_document: object, + qualified_action: bytes, + qualified_release: bytes, + hashes: Path, + wheel: Path, + sdist: Path, + version: str, + source_revision: str, + original_run_id: str, + controller_source: str, + reconciliation_run_id: str, + release_document: object, +) -> dict[str, object]: + _validate_original_run(run_document, run_id=original_run_id, source_revision=source_revision) + if qualified_action != qualified_release: + raise ExistingReleaseReconciliationError("draft release qualified receipt differs from the original run") + try: + qualified = ReleaseOperation.parse(json.loads(qualified_action)) + except (json.JSONDecodeError, ReleaseOperationError) as error: + raise ExistingReleaseReconciliationError("qualified release operation receipt is invalid") from error + + filenames = distribution_filenames(version) + expected = expected_digests_from_sha256sums(hashes, filenames) + observed = { + filenames[0]: _digest(wheel), + filenames[1]: _digest(sdist), + } + normalized_expected = {name: "sha256:" + digest for name, digest in expected.items()} + if observed != normalized_expected: + raise ExistingReleaseReconciliationError("fresh registry artifacts differ from the qualified hashes") + requested = ReleaseOperation.create( + operation_id=f"forge-release-{version}-{source_revision}", + version=version, + policy_revision="forge-bootstrap-release-cadence-v2", + source_revision=source_revision, + artifacts={"wheel": observed[filenames[0]], "sdist": observed[filenames[1]]}, + ) + if not qualified.same_identity(requested) or qualified.state != "QUALIFIED": + raise ExistingReleaseReconciliationError("qualified release receipt does not bind the requested identity") + qualification = dict(qualified.qualification or {}) + if qualification.get("exact_main_sha") != source_revision: + raise ExistingReleaseReconciliationError("qualification does not bind the product source") + if qualification.get("qualification") != "forge-production-distribution": + raise ExistingReleaseReconciliationError("qualification kind is not the production distribution gate") + if qualification.get("artifact_digests") != { + f"dist/{filenames[0]}": observed[filenames[0]], + f"dist/{filenames[1]}": observed[filenames[1]], + }: + raise ExistingReleaseReconciliationError("qualification does not bind the fresh registry artifacts") + _validate_package_metadata(wheel, sdist, version=version) + release_identity = validate_release_inventory( + release_document, version=version, source_revision=source_revision, + )["release"] + return { + "reconciliation_contract": RECONCILIATION_CONTRACT, + "original_release_run_id": original_run_id, + "original_release_run_conclusion": "failure", + "reconciliation_run_id": reconciliation_run_id, + "release_controller_source": controller_source, + "product_source_revision": source_revision, + "registry": "pypi", + "readback": "PASS", + "observed_artifact_digests": observed, + "github_release": release_identity, + } + + +def _release_asset_names(version: str, source_revision: str) -> dict[str, str]: + stem = f"{version}-{source_revision}.json" + return { + "qualified": f"forge-release-qualified-{stem}", + "published": f"forge-release-published-{stem}", + "cleanup_pending": f"forge-cleanup-pending-{stem}", + "complete": f"forge-release-complete-{stem}", + } + + +def validate_release_inventory( + document: object, *, version: str, source_revision: str, +) -> dict[str, object]: + if not isinstance(document, Mapping): + raise ExistingReleaseReconciliationError("GitHub release inventory is invalid") + expected_release = { + "tag": f"forge-v{version}", + "target_commitish": source_revision, + } + if document.get("tagName") != expected_release["tag"]: + raise ExistingReleaseReconciliationError("GitHub release tag name does not match") + if document.get("targetCommitish") != source_revision: + raise ExistingReleaseReconciliationError("GitHub release target does not match the product source") + database_id, node_id, api_url = document.get("databaseId"), document.get("id"), document.get("apiUrl") + if ( + not isinstance(database_id, int) or database_id <= 0 + or not isinstance(node_id, str) or not node_id + or not isinstance(api_url, str) or not api_url.startswith("https://api.github.com/") + ): + raise ExistingReleaseReconciliationError("GitHub release identity is incomplete") + if not isinstance(document.get("isDraft"), bool): + raise ExistingReleaseReconciliationError("GitHub release draft state is invalid") + if document.get("isPrerelease") is not False: + raise ExistingReleaseReconciliationError("production GitHub release must not be a prerelease") + if not isinstance(document.get("isImmutable"), bool): + raise ExistingReleaseReconciliationError("GitHub release immutability state is invalid") + assets = document.get("assets") + if not isinstance(assets, list) or any(not isinstance(item, Mapping) for item in assets): + raise ExistingReleaseReconciliationError("GitHub release asset inventory is invalid") + allowed = set(_release_asset_names(version, source_revision).values()) + names = [item.get("name") for item in assets] + if any(not isinstance(name, str) or name not in allowed for name in names): + raise ExistingReleaseReconciliationError("GitHub release contains an unexpected asset") + if len(names) != len(set(names)): + raise ExistingReleaseReconciliationError("GitHub release contains duplicate asset names") + if _release_asset_names(version, source_revision)["qualified"] not in names: + raise ExistingReleaseReconciliationError("GitHub release lacks the qualified operation receipt") + named = _release_asset_names(version, source_revision) + if named["complete"] in names and named["published"] not in names: + raise ExistingReleaseReconciliationError("GitHub release complete receipt lacks its published predecessor") + if named["cleanup_pending"] in names and named["published"] not in names: + raise ExistingReleaseReconciliationError("GitHub release cleanup-pending receipt lacks its published predecessor") + for item in assets: + if ( + item.get("state") != "uploaded" + or not isinstance(item.get("size"), int) or item["size"] <= 0 + or not isinstance(item.get("digest"), str) or _SHA256.fullmatch(item["digest"]) is None + ): + raise ExistingReleaseReconciliationError("GitHub release asset evidence is incomplete") + return { + "release": { + **expected_release, + "database_id": database_id, + "node_id": node_id, + "api_url": api_url, + }, + "is_draft": document["isDraft"], + "is_immutable": document["isImmutable"], + "assets": sorted(names), + } + + +def validate_same_release( + expected_document: object, current_document: object, *, version: str, source_revision: str, +) -> dict[str, object]: + expected = validate_release_inventory( + expected_document, version=version, source_revision=source_revision, + ) + current = validate_release_inventory( + current_document, version=version, source_revision=source_revision, + ) + if expected["release"] != current["release"]: + raise ExistingReleaseReconciliationError("GitHub release object identity changed during reconciliation") + return current + + +def validate_release_assets( + document: object, asset_root: Path, *, version: str, source_revision: str, +) -> dict[str, object]: + result = validate_release_inventory(document, version=version, source_revision=source_revision) + assets = document["assets"] # type: ignore[index] + expected_names = set(result["assets"]) # type: ignore[arg-type] + root = Path(asset_root) + if root.is_symlink() or not root.is_dir(): + raise ExistingReleaseReconciliationError("GitHub release asset directory is unavailable or unsafe") + observed_names = {path.name for path in root.iterdir()} + if observed_names != expected_names: + raise ExistingReleaseReconciliationError("downloaded GitHub release assets do not match inventory") + by_name = {str(item["name"]): item for item in assets} + for name in expected_names: + path = root / name + if path.is_symlink() or not path.is_file(): + raise ExistingReleaseReconciliationError("downloaded GitHub release asset is unavailable or unsafe") + if path.stat().st_size != by_name[name]["size"] or _digest(path) != by_name[name]["digest"]: + raise ExistingReleaseReconciliationError("downloaded GitHub release asset differs from inventory") + return result + + +def _validate_publication_evidence( + operation: ReleaseOperation, *, original_run_id: str, release_identity: Mapping[str, object], +) -> None: + expected_digests = { + f"forge_autonomy-{operation.version}-py3-none-any.whl": operation.artifacts["wheel"], + f"forge_autonomy-{operation.version}.tar.gz": operation.artifacts["sdist"], + } + evidence = operation.publication_receipt + if not isinstance(evidence, Mapping) or set(evidence) != { + "reconciliation_contract", "original_release_run_id", "original_release_run_conclusion", + "reconciliation_run_id", "release_controller_source", "product_source_revision", + "registry", "readback", "observed_artifact_digests", "github_release", + }: + raise ExistingReleaseReconciliationError("published receipt evidence shape is invalid") + if ( + evidence["reconciliation_contract"] != RECONCILIATION_CONTRACT + or evidence["original_release_run_id"] != original_run_id + or evidence["original_release_run_conclusion"] != "failure" + or evidence["product_source_revision"] != operation.source_revision + or evidence["registry"] != "pypi" + or evidence["readback"] != "PASS" + or evidence["observed_artifact_digests"] != expected_digests + or evidence["github_release"] != dict(release_identity) + or not isinstance(evidence["reconciliation_run_id"], str) + or _RUN_ID.fullmatch(evidence["reconciliation_run_id"]) is None + or not isinstance(evidence["release_controller_source"], str) + or _REVISION.fullmatch(evidence["release_controller_source"]) is None + ): + raise ExistingReleaseReconciliationError("published receipt evidence does not match the release identity") + + +def validate_new_publication_evidence( + *, qualified_path: Path, publication_path: Path, original_run_id: str, + release_identity: Mapping[str, object], controller_source: str, reconciliation_run_id: str, +) -> dict[str, object]: + try: + qualified = ReleaseOperation.parse(_load_json(qualified_path)) + except ReleaseOperationError as error: + raise ExistingReleaseReconciliationError("qualified operation receipt is invalid") from error + if qualified.state != "QUALIFIED": + raise ExistingReleaseReconciliationError("new publication must start from the qualified operation") + publication = _load_json(publication_path) + if not isinstance(publication, Mapping): + raise ExistingReleaseReconciliationError("new publication evidence is invalid") + try: + published = qualified.transition("PUBLISHED", evidence=publication) + except ReleaseOperationError as error: + raise ExistingReleaseReconciliationError("new publication evidence is invalid") from error + _validate_publication_evidence( + published, original_run_id=original_run_id, release_identity=release_identity, + ) + if ( + publication.get("release_controller_source") != controller_source + or publication.get("reconciliation_run_id") != reconciliation_run_id + ): + raise ExistingReleaseReconciliationError("new publication evidence is not bound to this controller run") + return { + "state": "QUALIFIED", + "controller_runs": [{ + "phase": "publication", "run_id": reconciliation_run_id, + "controller_source": controller_source, + }], + } + + +def validate_controller_run( + document: object, *, run_id: str, controller_source: str, +) -> dict[str, object]: + if not isinstance(document, Mapping): + raise ExistingReleaseReconciliationError("release controller run evidence is invalid") + expected = { + "databaseId": int(run_id), + "headSha": controller_source, + "event": "workflow_dispatch", + "workflowName": "Forge existing release reconciliation", + } + for key, value in expected.items(): + if document.get(key) != value: + raise ExistingReleaseReconciliationError(f"release controller run {key} does not match") + if document.get("status") not in {"queued", "in_progress", "completed"}: + raise ExistingReleaseReconciliationError("release controller run status is invalid") + return {"run_id": run_id, "controller_source": controller_source} + + +def validate_followup_receipts( + *, + published_path: Path, + pending_path: Path | None, + complete_path: Path | None, + original_run_id: str, + release_identity: Mapping[str, object], +) -> dict[str, object]: + try: + published = ReleaseOperation.parse(_load_json(published_path)) + except ReleaseOperationError as error: + raise ExistingReleaseReconciliationError("published follow-up receipt is invalid") from error + if published.state != "PUBLISHED": + raise ExistingReleaseReconciliationError("published follow-up receipt has the wrong state") + _validate_publication_evidence( + published, original_run_id=original_run_id, release_identity=release_identity, + ) + controller_runs = [{ + "phase": "publication", + "run_id": published.publication_receipt["reconciliation_run_id"], # type: ignore[index] + "controller_source": published.publication_receipt["release_controller_source"], # type: ignore[index] + }] + highest = published + pending: ReleaseOperation | None = None + if pending_path is not None: + try: + pending = ReleaseOperation.parse(_load_json(pending_path)) + except ReleaseOperationError as error: + raise ExistingReleaseReconciliationError("cleanup-pending follow-up receipt is invalid") from error + if ( + pending.state != "CLEANUP_PENDING" + or not published.same_identity(pending) + or pending.qualification != published.qualification + or pending.publication_receipt != published.publication_receipt + or not isinstance(pending.cleanup, Mapping) + or set(pending.cleanup) != { + "result", "reconciliation_contract", "original_release_run_id", + "reconciliation_run_id", "release_controller_source", "github_release", + "operation_local_cleanup", "failed_targets", + } + or pending.cleanup["result"] != "CLEANUP_PENDING" + or pending.cleanup["reconciliation_contract"] != RECONCILIATION_CONTRACT + or pending.cleanup["original_release_run_id"] != original_run_id + or pending.cleanup["github_release"] != { + **dict(release_identity), "draft": False, "tag_commit": published.source_revision, + } + or pending.cleanup["operation_local_cleanup"] != "PENDING" + or not isinstance(pending.cleanup["failed_targets"], list) + or not pending.cleanup["failed_targets"] + or any(not isinstance(item, str) or not item for item in pending.cleanup["failed_targets"]) + or not isinstance(pending.cleanup["reconciliation_run_id"], str) + or _RUN_ID.fullmatch(pending.cleanup["reconciliation_run_id"]) is None + or not isinstance(pending.cleanup["release_controller_source"], str) + or _REVISION.fullmatch(pending.cleanup["release_controller_source"]) is None + ): + raise ExistingReleaseReconciliationError("cleanup-pending follow-up receipt breaks release lineage") + highest = pending + controller_runs.append({ + "phase": "cleanup_pending", + "run_id": pending.cleanup["reconciliation_run_id"], # type: ignore[index] + "controller_source": pending.cleanup["release_controller_source"], # type: ignore[index] + }) + if complete_path is not None: + try: + complete = ReleaseOperation.parse(_load_json(complete_path)) + except ReleaseOperationError as error: + raise ExistingReleaseReconciliationError("complete follow-up receipt is invalid") from error + cleanup = complete.cleanup + if ( + complete.state != "RELEASE_COMPLETE" + or not published.same_identity(complete) + or complete.qualification != published.qualification + or complete.publication_receipt != published.publication_receipt + or not isinstance(cleanup, Mapping) + or set(cleanup) != { + "result", "reconciliation_contract", "original_release_run_id", + "reconciliation_run_id", "release_controller_source", "github_release", + "operation_local_cleanup", + } + or cleanup["result"] != "COMPLETE" + or cleanup["reconciliation_contract"] != RECONCILIATION_CONTRACT + or cleanup["original_release_run_id"] != original_run_id + or cleanup["operation_local_cleanup"] != "COMPLETE" + or cleanup["github_release"] != { + **dict(release_identity), "draft": False, "tag_commit": published.source_revision, + } + or not isinstance(cleanup["reconciliation_run_id"], str) + or _RUN_ID.fullmatch(cleanup["reconciliation_run_id"]) is None + or not isinstance(cleanup["release_controller_source"], str) + or _REVISION.fullmatch(cleanup["release_controller_source"]) is None + ): + raise ExistingReleaseReconciliationError("complete follow-up receipt breaks release lineage") + if pending is not None and ( + not pending.same_identity(complete) + or pending.qualification != complete.qualification + or pending.publication_receipt != complete.publication_receipt + ): + raise ExistingReleaseReconciliationError("complete receipt does not preserve cleanup-pending lineage") + highest = complete + controller_runs.append({ + "phase": "completion", + "run_id": complete.cleanup["reconciliation_run_id"], # type: ignore[index] + "controller_source": complete.cleanup["release_controller_source"], # type: ignore[index] + }) + return { + "state": highest.state, + "receipt": str(complete_path if complete_path is not None else pending_path or published_path), + "controller_runs": controller_runs, + } + + +def hydrate_followup_receipt(evidence_root: Path, operation_id: str, receipt_path: Path) -> ReleaseOperation: + try: + candidate = ReleaseOperation.parse(_load_json(receipt_path)) + except ReleaseOperationError as error: + raise ExistingReleaseReconciliationError("follow-up release receipt is invalid") from error + if candidate.operation_id != operation_id or candidate.state not in { + "PUBLISHED", "CLEANUP_PENDING", "RELEASE_COMPLETE", + }: + raise ExistingReleaseReconciliationError("follow-up release receipt has an invalid operation or state") + store = ReleaseOperationStore(evidence_root) + store.acquire(operation_id) + try: + current = store.load(operation_id) + if current is None or not current.same_identity(candidate): + raise ExistingReleaseReconciliationError("follow-up release receipt changes immutable identity") + if current.qualification != candidate.qualification: + raise ExistingReleaseReconciliationError("follow-up release receipt changes qualification evidence") + if _STATE_RANK[candidate.state] < _STATE_RANK[current.state]: + raise ExistingReleaseReconciliationError("follow-up release receipt regresses release state") + if current.publication_receipt is not None and current.publication_receipt != candidate.publication_receipt: + raise ExistingReleaseReconciliationError("follow-up release receipt changes publication evidence") + if current.cleanup is not None and current.cleanup != candidate.cleanup: + raise ExistingReleaseReconciliationError("follow-up release receipt changes completion evidence") + if current != candidate: + current = store.replace(current, candidate) + store.record_publication(current) + return current + finally: + store.release(operation_id) + + +def completion_evidence( + *, version: str, original_run_id: str, controller_source: str, reconciliation_run_id: str, + source_revision: str, release_identity: Mapping[str, object], +) -> dict[str, object]: + return { + "result": "COMPLETE", + "reconciliation_contract": RECONCILIATION_CONTRACT, + "original_release_run_id": original_run_id, + "reconciliation_run_id": reconciliation_run_id, + "release_controller_source": controller_source, + "github_release": { + **dict(release_identity), "tag": f"forge-v{version}", + "draft": False, "tag_commit": source_revision, + }, + "operation_local_cleanup": "COMPLETE", + } + + +def cleanup_pending_evidence( + *, original_run_id: str, controller_source: str, reconciliation_run_id: str, + source_revision: str, release_identity: Mapping[str, object], failed_targets: list[str], +) -> dict[str, object]: + if not failed_targets or any(not isinstance(item, str) or not item for item in failed_targets): + raise ExistingReleaseReconciliationError("cleanup-pending evidence requires failed targets") + return { + "result": "CLEANUP_PENDING", + "reconciliation_contract": RECONCILIATION_CONTRACT, + "original_release_run_id": original_run_id, + "reconciliation_run_id": reconciliation_run_id, + "release_controller_source": controller_source, + "github_release": { + **dict(release_identity), "draft": False, "tag_commit": source_revision, + }, + "operation_local_cleanup": "PENDING", + "failed_targets": failed_targets, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subcommands = parser.add_subparsers(dest="command", required=True) + validate = subcommands.add_parser("validate") + validate.add_argument("--run-json", type=Path, required=True) + validate.add_argument("--qualified-action", type=Path, required=True) + validate.add_argument("--qualified-release", type=Path, required=True) + validate.add_argument("--hashes", type=Path, required=True) + validate.add_argument("--wheel", type=Path, required=True) + validate.add_argument("--sdist", type=Path, required=True) + validate.add_argument("--version", required=True) + validate.add_argument("--source-revision", required=True) + validate.add_argument("--original-run-id", required=True) + validate.add_argument("--controller-source", required=True) + validate.add_argument("--reconciliation-run-id", required=True) + validate.add_argument("--release-json", type=Path, required=True) + inventory = subcommands.add_parser("inventory") + inventory.add_argument("--release-json", type=Path, required=True) + inventory.add_argument("--version", required=True) + inventory.add_argument("--source-revision", required=True) + assets = subcommands.add_parser("validate-assets") + assets.add_argument("--release-json", type=Path, required=True) + assets.add_argument("--asset-root", type=Path, required=True) + assets.add_argument("--version", required=True) + assets.add_argument("--source-revision", required=True) + same = subcommands.add_parser("same-release") + same.add_argument("--expected-release-json", type=Path, required=True) + same.add_argument("--current-release-json", type=Path, required=True) + same.add_argument("--version", required=True) + same.add_argument("--source-revision", required=True) + followup = subcommands.add_parser("validate-followup") + followup.add_argument("--published", type=Path, required=True) + followup.add_argument("--pending", type=Path) + followup.add_argument("--complete", type=Path) + followup.add_argument("--original-run-id", required=True) + followup.add_argument("--release-json", type=Path, required=True) + publication = subcommands.add_parser("validate-publication") + publication.add_argument("--qualified", type=Path, required=True) + publication.add_argument("--publication", type=Path, required=True) + publication.add_argument("--original-run-id", required=True) + publication.add_argument("--controller-source", required=True) + publication.add_argument("--reconciliation-run-id", required=True) + publication.add_argument("--release-json", type=Path, required=True) + controller = subcommands.add_parser("validate-controller-run") + controller.add_argument("--run-json", type=Path, required=True) + controller.add_argument("--controller-source", required=True) + controller.add_argument("--reconciliation-run-id", required=True) + hydrate = subcommands.add_parser("hydrate") + hydrate.add_argument("--evidence-root", type=Path, required=True) + hydrate.add_argument("--operation-id", required=True) + hydrate.add_argument("--receipt", type=Path, required=True) + complete = subcommands.add_parser("completion-evidence") + complete.add_argument("--version", required=True) + complete.add_argument("--original-run-id", required=True) + complete.add_argument("--controller-source", required=True) + complete.add_argument("--reconciliation-run-id", required=True) + complete.add_argument("--source-revision", required=True) + complete.add_argument("--release-json", type=Path, required=True) + pending = subcommands.add_parser("cleanup-pending-evidence") + pending.add_argument("--original-run-id", required=True) + pending.add_argument("--controller-source", required=True) + pending.add_argument("--reconciliation-run-id", required=True) + pending.add_argument("--version", required=True) + pending.add_argument("--source-revision", required=True) + pending.add_argument("--release-json", type=Path, required=True) + pending.add_argument("--failed-target", action="append", required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + if args.command == "validate": + result = validate_reconciliation( + run_document=_load_json(args.run_json), + qualified_action=args.qualified_action.read_bytes(), + qualified_release=args.qualified_release.read_bytes(), + hashes=args.hashes, + wheel=args.wheel, + sdist=args.sdist, + version=args.version, + source_revision=args.source_revision, + original_run_id=args.original_run_id, + controller_source=args.controller_source, + reconciliation_run_id=args.reconciliation_run_id, + release_document=_load_json(args.release_json), + ) + elif args.command == "inventory": + result = validate_release_inventory( + _load_json(args.release_json), version=args.version, source_revision=args.source_revision, + ) + elif args.command == "validate-assets": + result = validate_release_assets( + _load_json(args.release_json), args.asset_root, + version=args.version, source_revision=args.source_revision, + ) + elif args.command == "same-release": + result = validate_same_release( + _load_json(args.expected_release_json), _load_json(args.current_release_json), + version=args.version, source_revision=args.source_revision, + ) + elif args.command == "validate-followup": + try: + published = ReleaseOperation.parse(_load_json(args.published)) + except ReleaseOperationError as error: + raise ExistingReleaseReconciliationError("published follow-up receipt is invalid") from error + release = validate_release_inventory( + _load_json(args.release_json), version=published.version, + source_revision=published.source_revision, + )["release"] + result = validate_followup_receipts( + published_path=args.published, pending_path=args.pending, complete_path=args.complete, + original_run_id=args.original_run_id, release_identity=release, # type: ignore[arg-type] + ) + elif args.command == "validate-publication": + try: + qualified = ReleaseOperation.parse(_load_json(args.qualified)) + except ReleaseOperationError as error: + raise ExistingReleaseReconciliationError("qualified operation receipt is invalid") from error + release = validate_release_inventory( + _load_json(args.release_json), version=qualified.version, + source_revision=qualified.source_revision, + )["release"] + result = validate_new_publication_evidence( + qualified_path=args.qualified, publication_path=args.publication, + original_run_id=args.original_run_id, release_identity=release, # type: ignore[arg-type] + controller_source=args.controller_source, + reconciliation_run_id=args.reconciliation_run_id, + ) + elif args.command == "validate-controller-run": + result = validate_controller_run( + _load_json(args.run_json), run_id=args.reconciliation_run_id, + controller_source=args.controller_source, + ) + elif args.command == "hydrate": + result = asdict(hydrate_followup_receipt(args.evidence_root, args.operation_id, args.receipt)) + elif args.command == "completion-evidence": + result = completion_evidence( + version=args.version, + original_run_id=args.original_run_id, + controller_source=args.controller_source, + reconciliation_run_id=args.reconciliation_run_id, + source_revision=args.source_revision, + release_identity=validate_release_inventory( + _load_json(args.release_json), version=args.version, + source_revision=args.source_revision, + )["release"], # type: ignore[arg-type] + ) + else: + result = cleanup_pending_evidence( + original_run_id=args.original_run_id, + controller_source=args.controller_source, + reconciliation_run_id=args.reconciliation_run_id, + source_revision=args.source_revision, + release_identity=validate_release_inventory( + _load_json(args.release_json), version=args.version, + source_revision=args.source_revision, + )["release"], # type: ignore[arg-type] + failed_targets=args.failed_target, + ) + print(json.dumps(result, sort_keys=True, separators=(",", ":"), allow_nan=False)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ExistingReleaseReconciliationError as error: + raise SystemExit(f"EXISTING_RELEASE_RECONCILIATION_ERROR: {error}") from error diff --git a/tests/test_existing_release_reconciliation.py b/tests/test_existing_release_reconciliation.py new file mode 100644 index 0000000..805d498 --- /dev/null +++ b/tests/test_existing_release_reconciliation.py @@ -0,0 +1,357 @@ +"""Behavioral coverage for terminal reconciliation of an existing Forge release.""" + +import importlib.util +from hashlib import sha256 +import io +import json +from pathlib import Path +import sys +import tarfile +import tempfile +import unittest +import zipfile + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "reconcile_existing_release.py" +SPEC = importlib.util.spec_from_file_location("forge_existing_release_reconciliation", SCRIPT) +assert SPEC and SPEC.loader +reconcile = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = reconcile +SPEC.loader.exec_module(reconcile) + + +class ExistingReleaseReconciliationTests(unittest.TestCase): + version = "2.7.22" + source = "a" * 40 + original_run_id = "35254063298" + controller = "b" * 40 + reconciliation_run = "40000000000" + + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.wheel = self.root / f"forge_autonomy-{self.version}-py3-none-any.whl" + self.sdist = self.root / f"forge_autonomy-{self.version}.tar.gz" + metadata = f"Metadata-Version: 2.4\nName: forge-autonomy\nVersion: {self.version}\n\n" + with zipfile.ZipFile(self.wheel, "w") as archive: + archive.writestr(f"forge_autonomy-{self.version}.dist-info/METADATA", metadata) + with tarfile.open(self.sdist, "w:gz") as archive: + data = metadata.encode() + member = tarfile.TarInfo(f"forge_autonomy-{self.version}/PKG-INFO") + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + self.wheel_digest = sha256(self.wheel.read_bytes()).hexdigest() + self.sdist_digest = sha256(self.sdist.read_bytes()).hexdigest() + self.hashes = self.root / "SHA256SUMS" + self.hashes.write_text( + f"{self.wheel_digest} dist/{self.wheel.name}\n" + f"{self.sdist_digest} dist/{self.sdist.name}\n", + encoding="utf-8", + ) + self.operation = reconcile.ReleaseOperation.create( + operation_id=f"forge-release-{self.version}-{self.source}", + version=self.version, + policy_revision="forge-bootstrap-release-cadence-v2", + source_revision=self.source, + artifacts={ + "wheel": "sha256:" + self.wheel_digest, + "sdist": "sha256:" + self.sdist_digest, + }, + ).transition("QUALIFIED", evidence={ + "exact_main_sha": self.source, + "qualification": "forge-production-distribution", + "artifact_digests": { + f"dist/{self.wheel.name}": "sha256:" + self.wheel_digest, + f"dist/{self.sdist.name}": "sha256:" + self.sdist_digest, + }, + }) + self.qualified_bytes = self.operation_bytes(self.operation) + self.qualified_name = f"forge-release-qualified-{self.version}-{self.source}.json" + self.assets = self.root / "assets" + self.assets.mkdir() + (self.assets / self.qualified_name).write_bytes(self.qualified_bytes) + self.release = self.release_document([self.qualified_name]) + self.run = { + "databaseId": int(self.original_run_id), + "status": "completed", + "conclusion": "failure", + "headSha": self.source, + "event": "workflow_dispatch", + "workflowName": "Forge production release", + "jobs": [ + {"name": "build-and-qualify", "conclusion": "success"}, + {"name": "publish-pypi", "conclusion": "success"}, + {"name": "registry-readback-and-published-evidence", "conclusion": "failure"}, + {"name": "record-release-complete", "conclusion": "skipped"}, + ], + } + + @staticmethod + def operation_bytes(operation: object) -> bytes: + return (json.dumps(reconcile.asdict(operation), sort_keys=True, separators=(",", ":")) + "\n").encode() + + def release_document(self, names: list[str], *, draft: bool = True, database_id: int = 123) -> dict[str, object]: + assets = [] + for name in names: + content = (self.assets / name).read_bytes() + assets.append({ + "name": name, + "state": "uploaded", + "size": len(content), + "digest": "sha256:" + sha256(content).hexdigest(), + }) + return { + "tagName": f"forge-v{self.version}", + "targetCommitish": self.source, + "databaseId": database_id, + "id": "RE_kwDOReleaseNode", + "apiUrl": "https://api.github.com/repos/example/forge/releases/123", + "isDraft": draft, + "isPrerelease": False, + "isImmutable": False, + "assets": assets, + } + + @property + def release_identity(self) -> dict[str, object]: + return reconcile.validate_release_inventory( + self.release, version=self.version, source_revision=self.source, + )["release"] + + def validate(self, **changes: object) -> dict[str, object]: + values = { + "run_document": self.run, + "qualified_action": self.qualified_bytes, + "qualified_release": self.qualified_bytes, + "hashes": self.hashes, + "wheel": self.wheel, + "sdist": self.sdist, + "version": self.version, + "source_revision": self.source, + "original_run_id": self.original_run_id, + "controller_source": self.controller, + "reconciliation_run_id": self.reconciliation_run, + "release_document": self.release, + } + values.update(changes) + return reconcile.validate_reconciliation(**values) + + def published_operation(self) -> object: + return self.operation.transition("PUBLISHED", evidence=self.validate()) + + def test_exact_historical_release_and_fresh_registry_bytes_are_accepted(self) -> None: + evidence = self.validate() + self.assertEqual(evidence["readback"], "PASS") + self.assertEqual(evidence["product_source_revision"], self.source) + self.assertEqual(evidence["release_controller_source"], self.controller) + self.assertEqual(evidence["github_release"], self.release_identity) + + def test_wrong_artifact_or_unexpected_original_failure_is_rejected(self) -> None: + self.wheel.write_bytes(b"different registry bytes") + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "registry artifacts"): + self.validate() + changed_run = {**self.run, "conclusion": "success"} + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "conclusion"): + self.validate(run_document=changed_run) + + def test_changed_release_receipt_and_release_object_are_rejected(self) -> None: + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "differs"): + self.validate(qualified_release=self.qualified_bytes + b" ") + changed = self.release_document([self.qualified_name], database_id=124) + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "identity changed"): + reconcile.validate_same_release( + self.release, changed, version=self.version, source_revision=self.source, + ) + + def test_inventory_rejects_unknown_duplicate_and_incomplete_lineage(self) -> None: + unknown = {**self.release, "assets": [*self.release["assets"], { + "name": "forge_autonomy-2.7.22-py3-none-any.whl", "state": "uploaded", + "size": 1, "digest": "sha256:" + "0" * 64, + }]} + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "unexpected asset"): + reconcile.validate_release_inventory(unknown, version=self.version, source_revision=self.source) + duplicate = {**self.release, "assets": [*self.release["assets"], self.release["assets"][0]]} + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "duplicate"): + reconcile.validate_release_inventory(duplicate, version=self.version, source_revision=self.source) + complete = f"forge-release-complete-{self.version}-{self.source}.json" + (self.assets / complete).write_text("{}", encoding="utf-8") + incomplete = self.release_document([self.qualified_name, complete]) + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "published predecessor"): + reconcile.validate_release_inventory(incomplete, version=self.version, source_revision=self.source) + prerelease = {**self.release, "isPrerelease": True} + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "must not be a prerelease"): + reconcile.validate_release_inventory(prerelease, version=self.version, source_revision=self.source) + + def test_release_asset_bytes_and_symlinks_are_fail_closed(self) -> None: + self.assertEqual( + reconcile.validate_release_assets( + self.release, self.assets, version=self.version, source_revision=self.source, + )["assets"], + [self.qualified_name], + ) + (self.assets / self.qualified_name).write_bytes(b"changed") + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "differs"): + reconcile.validate_release_assets( + self.release, self.assets, version=self.version, source_revision=self.source, + ) + (self.assets / self.qualified_name).unlink() + (self.assets / self.qualified_name).symlink_to(self.hashes) + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "unsafe"): + reconcile.validate_release_assets( + self.release, self.assets, version=self.version, source_revision=self.source, + ) + + def test_followup_lineage_is_stable_across_resume_controller_runs(self) -> None: + published = self.published_operation() + published_path = self.root / "published.json" + published_path.write_bytes(self.operation_bytes(published)) + complete_evidence = reconcile.completion_evidence( + version=self.version, original_run_id=self.original_run_id, + controller_source="c" * 40, reconciliation_run_id="50000000000", + source_revision=self.source, release_identity=self.release_identity, + ) + complete = published.transition("RELEASE_COMPLETE", evidence=complete_evidence) + complete_path = self.root / "complete.json" + complete_path.write_bytes(self.operation_bytes(complete)) + result = reconcile.validate_followup_receipts( + published_path=published_path, pending_path=None, complete_path=complete_path, + original_run_id=self.original_run_id, release_identity=self.release_identity, + ) + self.assertEqual(result["state"], "RELEASE_COMPLETE") + self.assertEqual(result["controller_runs"], [ + {"phase": "publication", "run_id": self.reconciliation_run, + "controller_source": self.controller}, + {"phase": "completion", "run_id": "50000000000", + "controller_source": "c" * 40}, + ]) + self.assertEqual( + reconcile.validate_followup_receipts( + published_path=published_path, pending_path=None, complete_path=complete_path, + original_run_id=self.original_run_id, release_identity=self.release_identity, + ), result, + ) + + def test_cleanup_pending_is_durable_and_may_advance_to_complete(self) -> None: + published = self.published_operation() + published_path = self.root / "published.json" + published_path.write_bytes(self.operation_bytes(published)) + pending_evidence = reconcile.cleanup_pending_evidence( + original_run_id=self.original_run_id, controller_source="c" * 40, + reconciliation_run_id="50000000000", source_revision=self.source, + release_identity=self.release_identity, failed_targets=["published-assets"], + ) + pending = published.transition("CLEANUP_PENDING", evidence=pending_evidence) + pending_path = self.root / "pending.json" + pending_path.write_bytes(self.operation_bytes(pending)) + result = reconcile.validate_followup_receipts( + published_path=published_path, pending_path=pending_path, complete_path=None, + original_run_id=self.original_run_id, release_identity=self.release_identity, + ) + self.assertEqual(result["state"], "CLEANUP_PENDING") + complete = pending.transition("RELEASE_COMPLETE", evidence=reconcile.completion_evidence( + version=self.version, original_run_id=self.original_run_id, + controller_source="d" * 40, reconciliation_run_id="60000000000", + source_revision=self.source, release_identity=self.release_identity, + )) + complete_path = self.root / "complete.json" + complete_path.write_bytes(self.operation_bytes(complete)) + result = reconcile.validate_followup_receipts( + published_path=published_path, pending_path=pending_path, complete_path=complete_path, + original_run_id=self.original_run_id, release_identity=self.release_identity, + ) + self.assertEqual(result["state"], "RELEASE_COMPLETE") + + def test_new_publication_and_controller_run_bind_exact_current_provenance(self) -> None: + qualified = self.root / "qualified.json" + qualified.write_bytes(self.qualified_bytes) + publication = self.root / "publication.json" + publication.write_text(json.dumps(self.validate()), encoding="utf-8") + result = reconcile.validate_new_publication_evidence( + qualified_path=qualified, publication_path=publication, + original_run_id=self.original_run_id, release_identity=self.release_identity, + controller_source=self.controller, reconciliation_run_id=self.reconciliation_run, + ) + self.assertEqual(result["controller_runs"][0]["run_id"], self.reconciliation_run) + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "this controller run"): + reconcile.validate_new_publication_evidence( + qualified_path=qualified, publication_path=publication, + original_run_id=self.original_run_id, release_identity=self.release_identity, + controller_source="f" * 40, reconciliation_run_id=self.reconciliation_run, + ) + run = { + "databaseId": int(self.reconciliation_run), "headSha": self.controller, + "event": "workflow_dispatch", "workflowName": "Forge existing release reconciliation", + "status": "completed", "conclusion": "success", + } + self.assertEqual( + reconcile.validate_controller_run( + run, run_id=self.reconciliation_run, controller_source=self.controller, + )["controller_source"], self.controller, + ) + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "headSha"): + reconcile.validate_controller_run( + run, run_id=self.reconciliation_run, controller_source="f" * 40, + ) + + def test_followup_receipt_hydration_is_monotonic_and_idempotent(self) -> None: + evidence_root = self.root / "evidence" + store = reconcile.ReleaseOperationStore(evidence_root) + store.acquire(self.operation.operation_id) + try: + store.save(self.operation) + finally: + store.release(self.operation.operation_id) + published = self.published_operation() + receipt = self.root / "published.json" + receipt.write_bytes(self.operation_bytes(published)) + self.assertEqual( + reconcile.hydrate_followup_receipt(evidence_root, self.operation.operation_id, receipt), published, + ) + self.assertEqual( + reconcile.hydrate_followup_receipt(evidence_root, self.operation.operation_id, receipt), published, + ) + receipt.write_bytes(self.qualified_bytes) + with self.assertRaisesRegex(reconcile.ExistingReleaseReconciliationError, "invalid operation or state"): + reconcile.hydrate_followup_receipt(evidence_root, self.operation.operation_id, receipt) + + def test_workflow_separates_artifact_execution_from_release_mutation(self) -> None: + workflow = Path(".github/workflows/forge-existing-release-reconcile.yml").read_text(encoding="utf-8") + validation = workflow.split(" validate-existing-release:", 1)[1].split( + " smoke-existing-release:", 1, + )[0] + smoke = workflow.split(" smoke-existing-release:", 1)[1].split( + " reconcile-existing-release:", 1, + )[0] + mutation = workflow.split(" reconcile-existing-release:", 1)[1] + self.assertIn("persist-credentials: false", workflow) + self.assertIn("contents: read", validation) + self.assertNotIn("pip install", validation) + self.assertNotIn("installed_smoke", validation) + self.assertNotIn("gh release edit", validation) + self.assertIn("forge-registry-wheel", smoke) + self.assertIn("env -i HOME=", smoke) + self.assertIn("--no-index --no-deps", smoke) + self.assertNotIn("upload-artifact", smoke) + self.assertNotIn("publication.json", smoke) + self.assertIn("contents: write", mutation) + self.assertNotIn("pip install", mutation) + self.assertNotIn("registry-readback", mutation) + self.assertNotIn("forge-release-smoke-", mutation) + self.assertIn("validate-publication", mutation) + self.assertIn("validate-controller-run", mutation) + self.assertIn("verify_controller_runs", mutation) + self.assertIn("same-release", mutation) + self.assertIn("validate-assets", mutation) + self.assertIn("$tag^{commit}", mutation) + self.assertIn("--mark-cleanup-pending", mutation) + self.assertIn("--wheel-digest", mutation) + self.assertIn("--complete", mutation) + self.assertNotIn("gh-action-pypi-publish", workflow) + self.assertNotIn("twine upload", workflow) + self.assertNotIn("python3 -m build", workflow) + self.assertNotIn("git tag", workflow) + + +if __name__ == "__main__": + unittest.main()