From ea984f1e3a2de3cc607ae78dbcd4c5bceac9d4cb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Nov 2025 17:37:48 +0000 Subject: [PATCH 1/2] fix(ci): add retry logic with exponential backoff for image signing Resolves timing issues when signing images in consolidated workflow. The registry needs time to propagate multi-platform manifests after push. Changes: - Add retry logic (5 attempts with exponential backoff) to all signing steps - Add retry logic to all SBOM attestation steps - Improved error handling and debugging output - Initial delay: 2s, doubles each retry (2s, 4s, 8s, 16s, 32s) Root cause: When build/sign run in same workflow without delays, the manifest may not be immediately available for inspection via imagetools. Separate workflows had natural timing gaps that allowed propagation. This fix addresses failures like: - 'docker buildx imagetools inspect' returning incomplete/truncated digests - Cosign unable to reference images with malformed digest strings --- .github/workflows/container-images.yml | 192 ++++++++++++++++++++----- 1 file changed, 153 insertions(+), 39 deletions(-) diff --git a/.github/workflows/container-images.yml b/.github/workflows/container-images.yml index b1c90b8c..63de2a62 100644 --- a/.github/workflows/container-images.yml +++ b/.github/workflows/container-images.yml @@ -103,17 +103,33 @@ jobs: IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) echo "Image tag: $IMAGE_TAG" - # Use imagetools to get the manifest digest (works reliably for multi-platform builds) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}') - echo "Extracted digest: $DIGEST" - - # Validate digest format - if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "ERROR: unexpected digest format: $DIGEST" - echo "Full imagetools output:" - docker buildx imagetools inspect "$IMAGE_TAG" - exit 1 - fi + # Retry logic for manifest propagation + MAX_RETRIES=5 + RETRY_DELAY=2 + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt $i of $MAX_RETRIES to fetch manifest digest..." + + # Use imagetools to get the manifest digest (works reliably for multi-platform builds) + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + # Validate digest format + if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "✅ Successfully extracted digest: $DIGEST" + break + else + echo "⚠️ Invalid digest format (attempt $i): $DIGEST" + if [ $i -eq $MAX_RETRIES ]; then + echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" + echo "Full imagetools output:" + docker buildx imagetools inspect "$IMAGE_TAG" + exit 1 + fi + echo "Waiting ${RETRY_DELAY}s before retry..." + sleep $RETRY_DELAY + RETRY_DELAY=$((RETRY_DELAY * 2)) # Exponential backoff + fi + done IMAGE_REF="${{ env.IMAGE_PREFIX }}-controller@${DIGEST}" echo "Image reference for signing: $IMAGE_REF" @@ -134,9 +150,31 @@ jobs: COSIGN_EXPERIMENTAL: "true" run: | IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}') - echo "Using digest for SBOM attestation: $DIGEST" + + # Retry logic for manifest propagation + MAX_RETRIES=5 + RETRY_DELAY=2 + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt $i of $MAX_RETRIES to fetch manifest digest for SBOM attestation..." + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "✅ Successfully extracted digest: $DIGEST" + break + else + if [ $i -eq $MAX_RETRIES ]; then + echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" + exit 1 + fi + echo "Waiting ${RETRY_DELAY}s before retry..." + sleep $RETRY_DELAY + RETRY_DELAY=$((RETRY_DELAY * 2)) + fi + done + IMAGE_REF="${{ env.IMAGE_PREFIX }}-controller@${DIGEST}" + echo "Using digest for SBOM attestation: $DIGEST" cosign attest --yes --type spdxjson \ --predicate sbom-controller.spdx.json \ "$IMAGE_REF" @@ -222,17 +260,33 @@ jobs: IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) echo "Image tag: $IMAGE_TAG" - # Use imagetools to get the manifest digest (works reliably for multi-platform builds) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}') - echo "Extracted digest: $DIGEST" - - # Validate digest format - if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "ERROR: unexpected digest format: $DIGEST" - echo "Full imagetools output:" - docker buildx imagetools inspect "$IMAGE_TAG" - exit 1 - fi + # Retry logic for manifest propagation + MAX_RETRIES=5 + RETRY_DELAY=2 + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt $i of $MAX_RETRIES to fetch manifest digest..." + + # Use imagetools to get the manifest digest (works reliably for multi-platform builds) + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + # Validate digest format + if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "✅ Successfully extracted digest: $DIGEST" + break + else + echo "⚠️ Invalid digest format (attempt $i): $DIGEST" + if [ $i -eq $MAX_RETRIES ]; then + echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" + echo "Full imagetools output:" + docker buildx imagetools inspect "$IMAGE_TAG" + exit 1 + fi + echo "Waiting ${RETRY_DELAY}s before retry..." + sleep $RETRY_DELAY + RETRY_DELAY=$((RETRY_DELAY * 2)) # Exponential backoff + fi + done IMAGE_REF="${{ env.IMAGE_PREFIX }}-api@${DIGEST}" echo "Image reference for signing: $IMAGE_REF" @@ -253,9 +307,31 @@ jobs: COSIGN_EXPERIMENTAL: "true" run: | IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}') - echo "Using digest for SBOM attestation: $DIGEST" + + # Retry logic for manifest propagation + MAX_RETRIES=5 + RETRY_DELAY=2 + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt $i of $MAX_RETRIES to fetch manifest digest for SBOM attestation..." + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "✅ Successfully extracted digest: $DIGEST" + break + else + if [ $i -eq $MAX_RETRIES ]; then + echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" + exit 1 + fi + echo "Waiting ${RETRY_DELAY}s before retry..." + sleep $RETRY_DELAY + RETRY_DELAY=$((RETRY_DELAY * 2)) + fi + done + IMAGE_REF="${{ env.IMAGE_PREFIX }}-api@${DIGEST}" + echo "Using digest for SBOM attestation: $DIGEST" cosign attest --yes --type spdxjson \ --predicate sbom-api.spdx.json \ "$IMAGE_REF" @@ -341,17 +417,33 @@ jobs: IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) echo "Image tag: $IMAGE_TAG" - # Use imagetools to get the manifest digest (works reliably for multi-platform builds) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}') - echo "Extracted digest: $DIGEST" - - # Validate digest format - if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "ERROR: unexpected digest format: $DIGEST" - echo "Full imagetools output:" - docker buildx imagetools inspect "$IMAGE_TAG" - exit 1 - fi + # Retry logic for manifest propagation + MAX_RETRIES=5 + RETRY_DELAY=2 + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt $i of $MAX_RETRIES to fetch manifest digest..." + + # Use imagetools to get the manifest digest (works reliably for multi-platform builds) + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + # Validate digest format + if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "✅ Successfully extracted digest: $DIGEST" + break + else + echo "⚠️ Invalid digest format (attempt $i): $DIGEST" + if [ $i -eq $MAX_RETRIES ]; then + echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" + echo "Full imagetools output:" + docker buildx imagetools inspect "$IMAGE_TAG" + exit 1 + fi + echo "Waiting ${RETRY_DELAY}s before retry..." + sleep $RETRY_DELAY + RETRY_DELAY=$((RETRY_DELAY * 2)) # Exponential backoff + fi + done IMAGE_REF="${{ env.IMAGE_PREFIX }}-ui@${DIGEST}" echo "Image reference for signing: $IMAGE_REF" @@ -372,9 +464,31 @@ jobs: COSIGN_EXPERIMENTAL: "true" run: | IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}') - echo "Using digest for SBOM attestation: $DIGEST" + + # Retry logic for manifest propagation + MAX_RETRIES=5 + RETRY_DELAY=2 + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt $i of $MAX_RETRIES to fetch manifest digest for SBOM attestation..." + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "✅ Successfully extracted digest: $DIGEST" + break + else + if [ $i -eq $MAX_RETRIES ]; then + echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" + exit 1 + fi + echo "Waiting ${RETRY_DELAY}s before retry..." + sleep $RETRY_DELAY + RETRY_DELAY=$((RETRY_DELAY * 2)) + fi + done + IMAGE_REF="${{ env.IMAGE_PREFIX }}-ui@${DIGEST}" + echo "Using digest for SBOM attestation: $DIGEST" cosign attest --yes --type spdxjson \ --predicate sbom-ui.spdx.json \ "$IMAGE_REF" From 9e9843600ab35250ddec7364280ed21c5dca8d86 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Nov 2025 17:39:43 +0000 Subject: [PATCH 2/2] fix(ci): improve digest extraction for multi-platform image signing Use docker manifest inspect with jq as primary method, fallback to grep-based extraction from imagetools output. This resolves the issue where --format template returned the full manifest instead of digest. Changes: - Use 'docker manifest inspect | jq .digest' for reliable extraction - Fallback to grep pattern matching on imagetools output - Keep retry logic with exponential backoff for manifest propagation - Applied to all 6 digest extraction points (3 sign + 3 attest steps) Fixes error: 'ERROR: unexpected digest format: Name: ghcr.io/...' Root cause: The --format '{{.Manifest.Digest}}' template doesn't work reliably with docker buildx imagetools for multi-platform manifests. Using manifest inspect with JSON parsing is more reliable. --- .github/workflows/container-images.yml | 66 ++++++++++++++++++++------ 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/.github/workflows/container-images.yml b/.github/workflows/container-images.yml index 63de2a62..83f6a28e 100644 --- a/.github/workflows/container-images.yml +++ b/.github/workflows/container-images.yml @@ -110,8 +110,14 @@ jobs: for i in $(seq 1 $MAX_RETRIES); do echo "Attempt $i of $MAX_RETRIES to fetch manifest digest..." - # Use imagetools to get the manifest digest (works reliably for multi-platform builds) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + # Get the manifest digest using docker manifest inspect + # This returns the actual registry digest for multi-platform manifests + DIGEST=$(docker manifest inspect "$IMAGE_TAG" 2>&1 | jq -r '.digest // empty' || echo "") + + # If jq didn't find .digest, try extracting from Docker-Content-Digest header + if [ -z "$DIGEST" ]; then + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" 2>&1 | grep -oP 'Digest:\s*\K(sha256:[a-f0-9]{64})' | head -n1 || echo "") + fi # Validate digest format if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then @@ -121,8 +127,10 @@ jobs: echo "⚠️ Invalid digest format (attempt $i): $DIGEST" if [ $i -eq $MAX_RETRIES ]; then echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" - echo "Full imagetools output:" - docker buildx imagetools inspect "$IMAGE_TAG" + echo "Full docker manifest inspect output:" + docker manifest inspect "$IMAGE_TAG" || true + echo "Full buildx imagetools output:" + docker buildx imagetools inspect "$IMAGE_TAG" || true exit 1 fi echo "Waiting ${RETRY_DELAY}s before retry..." @@ -157,7 +165,11 @@ jobs: for i in $(seq 1 $MAX_RETRIES); do echo "Attempt $i of $MAX_RETRIES to fetch manifest digest for SBOM attestation..." - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + DIGEST=$(docker manifest inspect "$IMAGE_TAG" 2>&1 | jq -r '.digest // empty' || echo "") + if [ -z "$DIGEST" ]; then + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" 2>&1 | grep -oP 'Digest:\s*\K(sha256:[a-f0-9]{64})' | head -n1 || echo "") + fi if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then echo "✅ Successfully extracted digest: $DIGEST" @@ -267,8 +279,14 @@ jobs: for i in $(seq 1 $MAX_RETRIES); do echo "Attempt $i of $MAX_RETRIES to fetch manifest digest..." - # Use imagetools to get the manifest digest (works reliably for multi-platform builds) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + # Get the manifest digest using docker manifest inspect + # This returns the actual registry digest for multi-platform manifests + DIGEST=$(docker manifest inspect "$IMAGE_TAG" 2>&1 | jq -r '.digest // empty' || echo "") + + # If jq didn't find .digest, try extracting from Docker-Content-Digest header + if [ -z "$DIGEST" ]; then + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" 2>&1 | grep -oP 'Digest:\s*\K(sha256:[a-f0-9]{64})' | head -n1 || echo "") + fi # Validate digest format if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then @@ -278,8 +296,10 @@ jobs: echo "⚠️ Invalid digest format (attempt $i): $DIGEST" if [ $i -eq $MAX_RETRIES ]; then echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" - echo "Full imagetools output:" - docker buildx imagetools inspect "$IMAGE_TAG" + echo "Full docker manifest inspect output:" + docker manifest inspect "$IMAGE_TAG" || true + echo "Full buildx imagetools output:" + docker buildx imagetools inspect "$IMAGE_TAG" || true exit 1 fi echo "Waiting ${RETRY_DELAY}s before retry..." @@ -314,7 +334,11 @@ jobs: for i in $(seq 1 $MAX_RETRIES); do echo "Attempt $i of $MAX_RETRIES to fetch manifest digest for SBOM attestation..." - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + DIGEST=$(docker manifest inspect "$IMAGE_TAG" 2>&1 | jq -r '.digest // empty' || echo "") + if [ -z "$DIGEST" ]; then + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" 2>&1 | grep -oP 'Digest:\s*\K(sha256:[a-f0-9]{64})' | head -n1 || echo "") + fi if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then echo "✅ Successfully extracted digest: $DIGEST" @@ -424,8 +448,14 @@ jobs: for i in $(seq 1 $MAX_RETRIES); do echo "Attempt $i of $MAX_RETRIES to fetch manifest digest..." - # Use imagetools to get the manifest digest (works reliably for multi-platform builds) - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + # Get the manifest digest using docker manifest inspect + # This returns the actual registry digest for multi-platform manifests + DIGEST=$(docker manifest inspect "$IMAGE_TAG" 2>&1 | jq -r '.digest // empty' || echo "") + + # If jq didn't find .digest, try extracting from Docker-Content-Digest header + if [ -z "$DIGEST" ]; then + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" 2>&1 | grep -oP 'Digest:\s*\K(sha256:[a-f0-9]{64})' | head -n1 || echo "") + fi # Validate digest format if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then @@ -435,8 +465,10 @@ jobs: echo "⚠️ Invalid digest format (attempt $i): $DIGEST" if [ $i -eq $MAX_RETRIES ]; then echo "ERROR: Failed to get valid digest after $MAX_RETRIES attempts" - echo "Full imagetools output:" - docker buildx imagetools inspect "$IMAGE_TAG" + echo "Full docker manifest inspect output:" + docker manifest inspect "$IMAGE_TAG" || true + echo "Full buildx imagetools output:" + docker buildx imagetools inspect "$IMAGE_TAG" || true exit 1 fi echo "Waiting ${RETRY_DELAY}s before retry..." @@ -471,7 +503,11 @@ jobs: for i in $(seq 1 $MAX_RETRIES); do echo "Attempt $i of $MAX_RETRIES to fetch manifest digest for SBOM attestation..." - DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{.Manifest.Digest}}' 2>&1) + + DIGEST=$(docker manifest inspect "$IMAGE_TAG" 2>&1 | jq -r '.digest // empty' || echo "") + if [ -z "$DIGEST" ]; then + DIGEST=$(docker buildx imagetools inspect "$IMAGE_TAG" 2>&1 | grep -oP 'Digest:\s*\K(sha256:[a-f0-9]{64})' | head -n1 || echo "") + fi if [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then echo "✅ Successfully extracted digest: $DIGEST"