From 791119a5929fb7ed3427de75ba3d7f0b17353bbf Mon Sep 17 00:00:00 2001 From: thatbagu Date: Fri, 20 Nov 2026 21:42:29 +0500 Subject: [PATCH 1/2] ci: retry registry download flakes, cache the VM tests' images The VM tests' container images come from registries that time out now and then (ecr-public's redis failed several PR runs in a row), and the per-job store cache (3 GB) doesn't keep them. The build now retries on registry failures as it does on GitHub's, and image downloads go to the binary cache with the charts. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e2e65f..a23808f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,8 +74,9 @@ jobs: echo "Building: $(tr '\n' ' ' < checks.txt)" - name: Build checks # Helm charts download from GitHub release pages, which time out now - # and then (504). Retry only for that: what downloaded stays in the - # store, and real failures still fail on the first attempt. + # and then (504), and the VM tests' images from registries, which + # do too. Retry only for those: what downloaded stays in the store, + # and real failures still fail on the first attempt. run: | for attempt in 1 2 3; do set +e @@ -85,14 +86,14 @@ jobs: status=${PIPESTATUS[0]} set -e [ "$status" = 0 ] && exit 0 - grep -qE 'failed to fetch https://github\.com/[^ ]+ : 5[0-9][0-9]' build.log || exit "$status" - echo "::warning::A GitHub release download failed; retrying (attempt $attempt of 3)" + grep -qE 'failed to fetch https://github\.com/[^ ]+ : 5[0-9][0-9]|initializing source docker://|pinging container registry|reading manifest .*(timeout|EOF|50[0-9])' build.log || exit "$status" + echo "::warning::A chart or image download failed; retrying (attempt $attempt of 3)" sleep 30 done exit 1 - # Cache new chart downloads (only those: everything else is rebuilt - # cheaply), even when the build failed further on. - - name: Push chart downloads to the binary cache + # Cache new chart and image downloads (only those: everything else is + # rebuilt cheaply), even when the build failed further on. + - name: Push chart and image downloads to the binary cache if: always() env: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} @@ -101,8 +102,8 @@ jobs: [ -n "$CACHIX_AUTH_TOKEN" ] || { echo "No CACHIX_AUTH_TOKEN; not pushing."; exit 0; } shopt -s nullglob charts=() - for p in /nix/store/*-helm-chart-*; do - [[ "$(basename "$p")" =~ ^[a-z0-9]{32}-helm-chart- && "$p" != *.drv ]] && charts+=("$p") + for p in /nix/store/*-helm-chart-* /nix/store/*-docker-image-*.tar; do + [[ "$(basename "$p")" =~ ^[a-z0-9]{32}-(helm-chart|docker-image)- && "$p" != *.drv ]] && charts+=("$p") done [ ${#charts[@]} = 0 ] || cachix push thatbagu0nixlab "${charts[@]}" From db6bbf7fe5747929577bde563a1f3ee6d0f0be7c Mon Sep 17 00:00:00 2001 From: thatbagu Date: Fri, 20 Nov 2026 22:16:25 +0500 Subject: [PATCH 2/2] ci: retry any image download failure quay.io failed mid-blob (unexpected EOF), which the registry patterns didn't cover. Any fatal error from an image fetch is retried now. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a23808f..35e40c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: status=${PIPESTATUS[0]} set -e [ "$status" = 0 ] && exit 0 - grep -qE 'failed to fetch https://github\.com/[^ ]+ : 5[0-9][0-9]|initializing source docker://|pinging container registry|reading manifest .*(timeout|EOF|50[0-9])' build.log || exit "$status" + grep -qE 'failed to fetch https://github\.com/[^ ]+ : 5[0-9][0-9]|docker-image-[^ ]*> FATA' build.log || exit "$status" echo "::warning::A chart or image download failed; retrying (attempt $attempt of 3)" sleep 30 done