From 5dfdb2886a33efc6a48523aaedae38a1582276b6 Mon Sep 17 00:00:00 2001 From: isabelle-galleberg Date: Thu, 27 Aug 2026 13:46:47 +0200 Subject: [PATCH 1/2] Add pipeline artifact cleanup workflow --- .../deployment.artifact-cleanup-schedule.yml | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .github/workflows/deployment.artifact-cleanup-schedule.yml diff --git a/.github/workflows/deployment.artifact-cleanup-schedule.yml b/.github/workflows/deployment.artifact-cleanup-schedule.yml new file mode 100644 index 00000000..5992f274 --- /dev/null +++ b/.github/workflows/deployment.artifact-cleanup-schedule.yml @@ -0,0 +1,178 @@ +### +# Pipeline artifact cleanup (scheduled) +# +# Deletes stale artifacts uploaded by package.s3.yml / package.s3.jar.yml to the +# -deployment-delivery-pipeline-artifacts bucket, so it doesn't grow +# unbounded and become expensive to store. +# +# Artifacts are grouped by their upload "directory" (repo[/working-dir][/s3-path]) +# and then by the branch they were built from, using the branch tag that is +# written to the object's `tags` metadata on upload. Within each group: +# - Trunk branches (main/master) keep the newest `trunk-revisions-to-keep` +# revisions. Older ones are deleted regardless of age. +# - All other branches keep the newest `branch-revisions-to-keep` revisions. +# Anything beyond that is only deleted once it is older than +# `branch-days-to-keep` days, so an active branch's older builds aren't +# deleted while it's still in use. +# +# Call this from a repo's own workflow with a `schedule:` trigger, the same way +# deployment.preview-cleanup-schedule.yml is used. By default it only cleans up +# artifacts under the calling repo's own prefix in the bucket. +### +on: + workflow_call: + inputs: + s3-bucket-name: + required: false + type: string + description: "Name of the S3 bucket to clean up. Defaults to '-deployment-delivery-pipeline-artifacts'." + default: "" + trunk-branches: + required: false + type: string + description: "Comma-separated list of branch names considered trunk." + default: "main,master" + trunk-revisions-to-keep: + required: false + type: number + description: "Minimum number of trunk revisions to keep before older ones are deleted." + default: 5 + branch-revisions-to-keep: + required: false + type: number + description: "Minimum number of revisions to keep per non-trunk branch." + default: 2 + branch-days-to-keep: + required: false + type: number + description: "Number of days to keep non-trunk branch revisions beyond branch-revisions-to-keep, before they're deleted." + default: 30 + aws-region: + required: false + type: string + description: "AWS region to use." + default: 'eu-west-1' + +name: Pipeline artifact cleanup + +permissions: + contents: read + id-token: write + +jobs: + cleanup-artifacts: + name: Delete Stale Pipeline Artifacts + runs-on: ubuntu-latest + environment: Service + steps: + - name: Authenticate with AWS + uses: aws-actions/configure-aws-credentials@v6 + with: + aws-region: ${{ inputs.aws-region }} + role-to-assume: "arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_DEPLOYMENT_ROLE_NAME }}" + + - name: Delete stale artifacts + env: + S3_BUCKET: ${{ inputs.s3-bucket-name != '' && inputs.s3-bucket-name || format('{0}-deployment-delivery-pipeline-artifacts', vars.AWS_ACCOUNT_ID) }} + REPO_NAME: ${{ github.repository }} + TRUNK_BRANCHES: ${{ inputs.trunk-branches }} + TRUNK_KEEP: ${{ inputs.trunk-revisions-to-keep }} + BRANCH_KEEP: ${{ inputs.branch-revisions-to-keep }} + BRANCH_DAYS: ${{ inputs.branch-days-to-keep }} + run: | + set -euo pipefail + + REPO_NAME_WITHOUT_OWNER="${REPO_NAME##*/}" + echo "Cleaning up s3://${S3_BUCKET}/${REPO_NAME_WITHOUT_OWNER}/" + + OBJECTS_JSON=$(aws s3api list-objects-v2 \ + --bucket "${S3_BUCKET}" \ + --prefix "${REPO_NAME_WITHOUT_OWNER}/" \ + --query 'Contents[].{Key:Key,LastModified:LastModified}' \ + --output json) + + if [ "$OBJECTS_JSON" = "null" ] || [ -z "$OBJECTS_JSON" ]; then + echo "No artifacts found under ${REPO_NAME_WITHOUT_OWNER}/" + exit 0 + fi + + # Enrich every object with its branch (read from the `tags` metadata + # written on upload, e.g. '["-SHA","-branch"]') and the + # directory it lives in, so revisions can be grouped correctly. + ENRICHED="[]" + OBJECT_COUNT=$(echo "$OBJECTS_JSON" | jq 'length') + for ((i=0; i/dev/null || echo "") + BRANCH=$(echo "$TAGS_RAW" | grep -oE '"[^"]+-branch"' | sed -E 's/^"//; s/-branch"$//' || true) + + if [ -z "$BRANCH" ]; then + echo "::warning::Could not determine branch for $KEY, skipping" + continue + fi + + ENRICHED=$(echo "$ENRICHED" | jq --arg key "$KEY" --arg dir "$DIR" --arg branch "$BRANCH" --arg lm "$LAST_MODIFIED" \ + '. += [{"key": $key, "dir": $dir, "branch": $branch, "lastModified": $lm}]') + done + + TO_DELETE="[]" + CURRENT_TIME=$(date +%s) + CUTOFF_SECONDS=$(( BRANCH_DAYS * 24 * 60 * 60 )) + + DIRS=$(echo "$ENRICHED" | jq -r '[.[].dir] | unique | .[]') + for DIR in $DIRS; do + # Trunk revisions: keep the newest TRUNK_KEEP, delete the rest regardless of age. + TRUNK_ITEMS=$(echo "$ENRICHED" | jq -c --arg dir "$DIR" --arg branches "$TRUNK_BRANCHES" \ + '($branches | split(",")) as $tb | [.[] | select(.dir == $dir) | select(.branch as $b | $tb | index($b) != null)] | sort_by(.lastModified) | reverse') + TRUNK_COUNT=$(echo "$TRUNK_ITEMS" | jq 'length') + if [ "$TRUNK_COUNT" -gt "$TRUNK_KEEP" ]; then + STALE=$(echo "$TRUNK_ITEMS" | jq -c ".[${TRUNK_KEEP}:]") + TO_DELETE=$(echo "$TO_DELETE" | jq --argjson stale "$STALE" '. + $stale') + fi + + # Branch revisions: keep the newest BRANCH_KEEP per branch, delete the + # rest only once older than BRANCH_DAYS days. + BRANCHES_IN_DIR=$(echo "$ENRICHED" | jq -r --arg dir "$DIR" --arg branches "$TRUNK_BRANCHES" \ + '($branches | split(",")) as $tb | [.[] | select(.dir == $dir) | select(.branch as $b | $tb | index($b) == null) | .branch] | unique | .[]') + for BRANCH in $BRANCHES_IN_DIR; do + BRANCH_ITEMS=$(echo "$ENRICHED" | jq -c --arg dir "$DIR" --arg branch "$BRANCH" \ + '[.[] | select(.dir == $dir and .branch == $branch)] | sort_by(.lastModified) | reverse') + BRANCH_COUNT=$(echo "$BRANCH_ITEMS" | jq 'length') + if [ "$BRANCH_COUNT" -le "$BRANCH_KEEP" ]; then + continue + fi + + CANDIDATES=$(echo "$BRANCH_ITEMS" | jq -c ".[${BRANCH_KEEP}:]") + CANDIDATE_COUNT=$(echo "$CANDIDATES" | jq 'length') + for ((j=0; j Date: Thu, 27 Aug 2026 15:35:16 +0200 Subject: [PATCH 2/2] Address PR review feedback on artifact cleanup workflow - Trim whitespace and drop empty entries when parsing trunk-branches, so "main, master" doesn't misclassify master as a branch - Batch object deletion via s3api delete-objects (up to 1000 keys per call) instead of one aws s3 rm per object, to avoid slow runs and API throttling when many artifacts are stale --- .../deployment.artifact-cleanup-schedule.yml | 45 ++++++++----------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/.github/workflows/deployment.artifact-cleanup-schedule.yml b/.github/workflows/deployment.artifact-cleanup-schedule.yml index 5992f274..0e1bf6b5 100644 --- a/.github/workflows/deployment.artifact-cleanup-schedule.yml +++ b/.github/workflows/deployment.artifact-cleanup-schedule.yml @@ -1,23 +1,9 @@ ### -# Pipeline artifact cleanup (scheduled) -# -# Deletes stale artifacts uploaded by package.s3.yml / package.s3.jar.yml to the -# -deployment-delivery-pipeline-artifacts bucket, so it doesn't grow -# unbounded and become expensive to store. -# -# Artifacts are grouped by their upload "directory" (repo[/working-dir][/s3-path]) -# and then by the branch they were built from, using the branch tag that is -# written to the object's `tags` metadata on upload. Within each group: -# - Trunk branches (main/master) keep the newest `trunk-revisions-to-keep` -# revisions. Older ones are deleted regardless of age. -# - All other branches keep the newest `branch-revisions-to-keep` revisions. -# Anything beyond that is only deleted once it is older than -# `branch-days-to-keep` days, so an active branch's older builds aren't -# deleted while it's still in use. -# -# Call this from a repo's own workflow with a `schedule:` trigger, the same way -# deployment.preview-cleanup-schedule.yml is used. By default it only cleans up -# artifacts under the calling repo's own prefix in the bucket. +# Deletes stale artifacts from the deployment-delivery-pipeline-artifacts +# bucket: trunk (main/master) keeps the newest `trunk-revisions-to-keep`, +# other branches keep `branch-revisions-to-keep` and expire after +# `branch-days-to-keep` days. Call from a repo's own `schedule:` workflow, +# like deployment.preview-cleanup-schedule.yml. ### on: workflow_call: @@ -134,7 +120,7 @@ jobs: for DIR in $DIRS; do # Trunk revisions: keep the newest TRUNK_KEEP, delete the rest regardless of age. TRUNK_ITEMS=$(echo "$ENRICHED" | jq -c --arg dir "$DIR" --arg branches "$TRUNK_BRANCHES" \ - '($branches | split(",")) as $tb | [.[] | select(.dir == $dir) | select(.branch as $b | $tb | index($b) != null)] | sort_by(.lastModified) | reverse') + '($branches | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))) as $tb | [.[] | select(.dir == $dir) | select(.branch as $b | $tb | index($b) != null)] | sort_by(.lastModified) | reverse') TRUNK_COUNT=$(echo "$TRUNK_ITEMS" | jq 'length') if [ "$TRUNK_COUNT" -gt "$TRUNK_KEEP" ]; then STALE=$(echo "$TRUNK_ITEMS" | jq -c ".[${TRUNK_KEEP}:]") @@ -144,7 +130,7 @@ jobs: # Branch revisions: keep the newest BRANCH_KEEP per branch, delete the # rest only once older than BRANCH_DAYS days. BRANCHES_IN_DIR=$(echo "$ENRICHED" | jq -r --arg dir "$DIR" --arg branches "$TRUNK_BRANCHES" \ - '($branches | split(",")) as $tb | [.[] | select(.dir == $dir) | select(.branch as $b | $tb | index($b) == null) | .branch] | unique | .[]') + '($branches | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))) as $tb | [.[] | select(.dir == $dir) | select(.branch as $b | $tb | index($b) == null) | .branch] | unique | .[]') for BRANCH in $BRANCHES_IN_DIR; do BRANCH_ITEMS=$(echo "$ENRICHED" | jq -c --arg dir "$DIR" --arg branch "$BRANCH" \ '[.[] | select(.dir == $dir and .branch == $branch)] | sort_by(.lastModified) | reverse') @@ -170,9 +156,14 @@ jobs: DELETE_COUNT=$(echo "$TO_DELETE" | jq 'length') echo "Found ${DELETE_COUNT} stale artifact(s) to delete" - echo "$TO_DELETE" | jq -c '.[]' | while read -r ITEM; do - KEY=$(echo "$ITEM" | jq -r '.key') - BRANCH=$(echo "$ITEM" | jq -r '.branch') - echo "Deleting s3://${S3_BUCKET}/${KEY} (branch: ${BRANCH})" - aws s3 rm "s3://${S3_BUCKET}/${KEY}" - done + if [ "$DELETE_COUNT" -gt 0 ]; then + echo "$TO_DELETE" | jq -r '.[] | " \(.key) (branch: \(.branch))"' + + # Delete in batches of up to 1000 keys, the max supported by delete-objects. + KEYS_JSON=$(echo "$TO_DELETE" | jq -c '[.[].key]') + for ((start=0; start