Fix iamb and packages batch #71
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Build COPR packages when package specs change | |
| # Triggers builds in dependency order based on packages/README.md | |
| name: Build COPR Packages | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false # Don't cancel - we need to cleanup COPR builds properly | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'packages/**' | |
| - '.github/workflows/build-copr-packages.yml' | |
| workflow_dispatch: | |
| inputs: | |
| packages: | |
| description: 'Comma-separated list of packages to build (empty = auto-detect from changes)' | |
| required: false | |
| type: string | |
| rebuild_all: | |
| description: 'Rebuild all packages in dependency order' | |
| required: false | |
| type: boolean | |
| default: false | |
| env: | |
| COPR_PROJECT: binarypie/hypercube | |
| jobs: | |
| detect-changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| packages: ${{ steps.detect.outputs.packages }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 # Full history to find last successful build | |
| - name: Get last successful build commit | |
| id: last-success | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Find the last successful workflow run on this branch | |
| LAST_SHA=$(gh run list \ | |
| --workflow="Build COPR Packages" \ | |
| --branch=main \ | |
| --status=success \ | |
| --limit=1 \ | |
| --json headSha \ | |
| --jq '.[0].headSha // empty' 2>/dev/null) || LAST_SHA="" | |
| if [[ -n "$LAST_SHA" ]]; then | |
| echo "Last successful build: $LAST_SHA" | |
| echo "sha=$LAST_SHA" >> $GITHUB_OUTPUT | |
| else | |
| echo "No previous successful build found, will use HEAD~1" | |
| echo "sha=" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Install jq | |
| run: sudo apt-get update && sudo apt-get install -y jq | |
| - name: Detect changed packages | |
| id: detect | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Source shared package configuration | |
| source ./scripts/packages/config.sh | |
| # Package dependencies are now available via PACKAGE_DEPS | |
| # Build batches are available via BUILD_BATCHES | |
| # Function to get all downstream dependents of a package | |
| get_dependents() { | |
| local pkg=$1 | |
| local dependents="" | |
| for p in "${!PACKAGE_DEPS[@]}"; do | |
| if [[ " ${PACKAGE_DEPS[$p]} " == *" $pkg "* ]]; then | |
| dependents="$dependents $p" | |
| # Recursively get dependents of dependents | |
| dependents="$dependents $(get_dependents $p)" | |
| fi | |
| done | |
| echo "$dependents" | |
| } | |
| # Determine which packages to build | |
| if [[ "${{ inputs.rebuild_all }}" == "true" ]]; then | |
| echo "Rebuilding all packages..." | |
| CHANGED_PKGS="${!PACKAGE_DEPS[*]}" | |
| elif [[ -n "${{ inputs.packages }}" ]]; then | |
| echo "Using manually specified packages..." | |
| CHANGED_PKGS=$(echo "${{ inputs.packages }}" | tr ',' ' ') | |
| else | |
| echo "Detecting changed packages..." | |
| # Compare against last successful build, or HEAD~1 if none | |
| LAST_SHA="${{ steps.last-success.outputs.sha }}" | |
| if [[ -n "$LAST_SHA" ]]; then | |
| echo "Comparing HEAD against last successful build: $LAST_SHA" | |
| COMPARE_REF="$LAST_SHA" | |
| else | |
| echo "Comparing HEAD against HEAD~1" | |
| COMPARE_REF="HEAD~1" | |
| fi | |
| CHANGED_PKGS="" | |
| for dir in packages/*/; do | |
| pkg=$(basename "$dir") | |
| if [[ "$pkg" != "README.md" ]]; then | |
| if git diff --name-only "$COMPARE_REF" HEAD | grep -q "^packages/$pkg/"; then | |
| CHANGED_PKGS="$CHANGED_PKGS $pkg" | |
| fi | |
| fi | |
| done | |
| fi | |
| echo "Initially changed packages: $CHANGED_PKGS" | |
| echo "" | |
| # rebuild_all bypasses the version filter so stale Copr builds | |
| # (same version, but linked against an older soname) get refreshed. | |
| if [[ "${{ inputs.rebuild_all }}" == "true" ]]; then | |
| echo "rebuild_all=true: skipping version check, forcing rebuild of all packages" | |
| NEEDS_BUILD="$CHANGED_PKGS" | |
| elif [[ -z "$CHANGED_PKGS" ]]; then | |
| NEEDS_BUILD="" | |
| else | |
| echo "Checking which packages actually need building..." | |
| echo "" | |
| # Run the version check script | |
| SCRIPT_OUTPUT=$(./scripts/packages/check-copr-versions.sh $CHANGED_PKGS 2>&1) | |
| echo "$SCRIPT_OUTPUT" | |
| # Extract packages that need building from the script output | |
| NEEDS_BUILD=$(echo "$SCRIPT_OUTPUT" | grep "^NEEDS_BUILD_JSON=" | sed 's/^NEEDS_BUILD_JSON=//' | jq -r '.[]' | tr '\n' ' ') | |
| fi | |
| echo "" | |
| echo "Packages that need building: $NEEDS_BUILD" | |
| # Add all downstream dependents | |
| TO_BUILD="$NEEDS_BUILD" | |
| for pkg in $NEEDS_BUILD; do | |
| dependents=$(get_dependents "$pkg") | |
| TO_BUILD="$TO_BUILD $dependents" | |
| done | |
| # Deduplicate | |
| TO_BUILD=$(echo "$TO_BUILD" | tr ' ' '\n' | sort -u | tr '\n' ' ') | |
| echo "Packages to build (including dependents): $TO_BUILD" | |
| # Output as JSON array | |
| if [[ -z "$(echo $TO_BUILD | tr -d ' ')" ]]; then | |
| echo "packages=[]" >> $GITHUB_OUTPUT | |
| else | |
| JSON=$(echo "$TO_BUILD" | tr -s ' ' '\n' | grep -v '^$' | jq -R . | jq -sc .) | |
| echo "packages=$JSON" >> $GITHUB_OUTPUT | |
| fi | |
| build: | |
| needs: detect-changes | |
| if: ${{ needs.detect-changes.outputs.packages != '[]' && needs.detect-changes.outputs.packages != '' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Install COPR CLI | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y python3-pip jq | |
| pip3 install copr-cli | |
| - name: Configure COPR CLI | |
| run: | | |
| mkdir -p ~/.config | |
| cat > ~/.config/copr << EOF | |
| [copr-cli] | |
| login = ${{ secrets.COPR_API_LOGIN }} | |
| username = binarypie | |
| token = ${{ secrets.COPR_API_TOKEN }} | |
| copr_url = https://copr.fedorainfracloud.org | |
| EOF | |
| # Submit every package up-front, using Copr's native batching to express | |
| # the dep graph: | |
| # --with-build-id -> same Copr batch (parallel execution) | |
| # --after-build-id -> new Copr batch blocked by parent (sequential) | |
| # Each of our BUILD_BATCHES layers maps to one Copr batch; layer N+1's | |
| # first build chains to layer N's first build via --after-build-id, and | |
| # the rest of layer N+1 joins it via --with-build-id. Copr then runs | |
| # everything as parallel as the dep graph allows, in one GH job. | |
| - name: Submit all builds with Copr batch chaining | |
| id: submit | |
| run: | | |
| source ./scripts/packages/config.sh | |
| PACKAGES_JSON='${{ needs.detect-changes.outputs.packages }}' | |
| ALL_BUILD_IDS="" | |
| PREV_LAYER_FIRST_ID="" | |
| # Iterate layers in defined order | |
| for layer_id in $(echo "${!BUILD_BATCHES[@]}" | tr ' ' '\n' | sort -n); do | |
| echo "" | |
| echo "========================================" | |
| echo "Layer $layer_id" | |
| echo "========================================" | |
| LAYER_FIRST_ID="" | |
| for pkg in ${BUILD_BATCHES[$layer_id]}; do | |
| # Skip packages not in the filtered build set | |
| if ! echo "$PACKAGES_JSON" | jq -e --arg p "$pkg" 'index($p) != null' >/dev/null; then | |
| continue | |
| fi | |
| ARGS=(--nowait --name "$pkg") | |
| if [[ -z "$LAYER_FIRST_ID" ]]; then | |
| # First package in this layer establishes the batch. | |
| # If a previous layer had builds, chain after it. | |
| if [[ -n "$PREV_LAYER_FIRST_ID" ]]; then | |
| ARGS+=(--after-build-id "$PREV_LAYER_FIRST_ID") | |
| fi | |
| else | |
| # Join the layer's existing batch (parallel with siblings) | |
| ARGS+=(--with-build-id "$LAYER_FIRST_ID") | |
| fi | |
| echo "Submitting $pkg (${ARGS[@]:1})" | |
| BUILD_OUTPUT=$(copr-cli build-package "${ARGS[@]}" "${{ env.COPR_PROJECT }}" 2>&1) || { | |
| echo "::error::Failed to submit $pkg" | |
| echo "$BUILD_OUTPUT" | |
| exit 1 | |
| } | |
| echo "$BUILD_OUTPUT" | |
| BUILD_ID=$(echo "$BUILD_OUTPUT" | grep -oP '^Created builds?:\s*\K\d+' | head -1) | |
| if [[ -z "$BUILD_ID" ]]; then | |
| echo "::error::Could not extract build ID for $pkg" | |
| exit 1 | |
| fi | |
| ALL_BUILD_IDS="$ALL_BUILD_IDS $BUILD_ID" | |
| [[ -z "$LAYER_FIRST_ID" ]] && LAYER_FIRST_ID="$BUILD_ID" | |
| done | |
| [[ -n "$LAYER_FIRST_ID" ]] && PREV_LAYER_FIRST_ID="$LAYER_FIRST_ID" | |
| done | |
| ALL_BUILD_IDS="$(echo "$ALL_BUILD_IDS" | xargs)" | |
| echo "" | |
| echo "All submitted build IDs: $ALL_BUILD_IDS" | |
| echo "build_ids=$ALL_BUILD_IDS" >> $GITHUB_OUTPUT | |
| - name: Watch all builds | |
| id: watch | |
| run: | | |
| FAILED="" | |
| # watch-build blocks per-build, but Copr runs them in parallel where | |
| # the batch graph allows, so total wall time = critical path of the | |
| # dep chain, not sum of individual builds. | |
| for build_id in ${{ steps.submit.outputs.build_ids }}; do | |
| echo "" | |
| echo "Watching build $build_id..." | |
| copr-cli watch-build "$build_id" || { | |
| echo "::warning::Build $build_id failed or was cancelled" | |
| FAILED="$FAILED $build_id" | |
| } | |
| done | |
| if [[ -n "$FAILED" ]]; then | |
| echo "::error::Failed build IDs:$FAILED" | |
| exit 1 | |
| fi | |
| - name: Cancel COPR builds on workflow cancellation | |
| if: cancelled() | |
| run: | | |
| echo "Workflow cancelled - cancelling COPR builds..." | |
| for build_id in ${{ steps.submit.outputs.build_ids }}; do | |
| echo "Cancelling COPR build $build_id..." | |
| copr-cli cancel "$build_id" || echo "Could not cancel $build_id (may have already finished)" | |
| done | |
| - name: Generate summary | |
| if: always() | |
| run: | | |
| echo "## COPR Package Build Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Packages built:** ${{ needs.detect-changes.outputs.packages }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Build IDs:** ${{ steps.submit.outputs.build_ids }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "[View all builds](https://copr.fedorainfracloud.org/coprs/${{ env.COPR_PROJECT }}/builds/)" >> $GITHUB_STEP_SUMMARY |