CBG-5862: retry Go module downloads in CI

.ci/retry.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# Copyright 2026-Present Couchbase, Inc.
+#
+# Use of this software is governed by the Business Source License included in
+# the file licenses/BSL-Couchbase.txt.  As of the Change Date specified in that
+# file, in accordance with the Business Source License, use of this software
+# will be governed by the Apache License, Version 2.0, included in the file
+# licenses/APL2.txt.
+
+# Runs a command, retrying with exponential backoff. CI uses this for Go module downloads, which
+# fail intermittently when the module proxy is unhealthy.
+#
+# Usage: .ci/retry.sh <command> [args...]
+# Tunable with RETRY_ATTEMPTS (default 4) and RETRY_DELAY (initial backoff seconds, default 5).
+
+set -euo pipefail
+
+attempts=${RETRY_ATTEMPTS:-4}
+delay=${RETRY_DELAY:-5}
+
+# without these guards, no args exits 0 without running anything and a bad attempt count skips the loop entirely
+if (($# == 0)); then
+    echo "retry: usage: .ci/retry.sh <command> [args...]" >&2
+    exit 2
+fi
+if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then
+    echo "retry: RETRY_ATTEMPTS must be a positive integer, got '${attempts}'" >&2
+    exit 2
+fi
+if [[ ! "${delay}" =~ ^[0-9]+$ ]]; then
+    echo "retry: RETRY_DELAY must be a non-negative integer, got '${delay}'" >&2
+    exit 2
+fi
+
+for ((attempt = 1; attempt <= attempts; attempt++)); do
+    # capture the status here: $? after an if-block is the status of the if, not of the command
+    status=0
+    "$@" || status=$?
+    if ((status == 0)); then
+        exit 0
+    fi
+    if ((attempt < attempts)); then
+        # diagnostics go to stderr so callers capturing stdout get the command's output only
+        echo "retry: attempt ${attempt}/${attempts} of '$*' failed (exit ${status}), retrying in ${delay}s" >&2
+        sleep "${delay}"
+        delay=$((delay * 2))
+    fi
+done
+
+echo "retry: '$*' failed after ${attempts} attempts" >&2
+exit "${status}"

.github/workflows/ci.yml

@@ -52,6 +52,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: go-build
         run: go build "./..."
 
@@ -63,6 +67,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - run: go tool addlicense -check -f licenses/addlicense.tmpl .
 
   golangci:
@@ -73,6 +81,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: golangci-lint
         uses: *actions_golangci_lint_version
         with:
@@ -100,6 +112,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: Build
         run: go build -v "./..."
       - name: Run Tests
@@ -136,6 +152,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: Run Tests
         run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -race -shuffle=on -timeout=30m -count=1 "./..."
         shell: bash
@@ -171,6 +191,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: Run Tests
         run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 "./..."
         shell: bash
@@ -204,6 +228,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: Run Tests
         run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 -short -bench=. -benchtime=1x -run '!' "./..."
         shell: bash
@@ -240,6 +268,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: Run Tests
         run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 "./..."
         shell: bash
@@ -304,6 +336,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: Build
         run: go build -v "./tools/stats-definition-exporter"
       - name: Run Tests
@@ -338,6 +374,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: *go_mod_file
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: Build
         run: go build "./tools/cache_perf_tool"
   govulncheck:

.github/workflows/service.yml

@@ -53,6 +53,10 @@ jobs:
       - uses: *actions_setup_go_version
         with:
           go-version-file: go.mod
+      # go mod download would try to fetch the private EE module, which public runners can't reach
+      - name: Download Go modules
+        shell: bash
+        run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null
       - name: "Build Sync Gateway"
         run: mkdir -p ./bin && go build -o ./bin ./...
       - name: "Run test scripts"

.github/workflows/shell.yml

@@ -15,6 +15,7 @@ on:
     # Only run when we change a shell script
     paths:
       - "*.sh"
+      - ".ci/*.sh"
     branches:
       - "main"
       - "release/*"
@@ -25,6 +26,7 @@ on:
     # Only run when we change a shell script
     paths:
       - "*.sh"
+      - ".ci/*.sh"
     branches:
       - "main"
       - "release/*"

Jenkinsfile

@@ -38,16 +38,16 @@ pipeline {
                             env.GO_VERSION = 'go' + sh(
                               returnStdout: true,
                               script: '''
-                                go list -m -f '{{.GoVersion}}'
+                                .ci/retry.sh go list -m -f '{{.GoVersion}}'
                               '''
                             ).trim()
                             // go fetches the toolchain named in go.mod on demand
                             sh '''
                               set -eux
 
                               echo "Sync Gateway go.mod version is $GO_VERSION"
-                              go install "golang.org/dl/$GO_VERSION@latest"
-                              ~/go/bin/$GO_VERSION download
+                              .ci/retry.sh go install "golang.org/dl/$GO_VERSION@latest"
+                              .ci/retry.sh ~/go/bin/$GO_VERSION download
                             '''
                             env.GOROOT = sh(
                               returnStdout: true,
@@ -68,7 +68,7 @@ pipeline {
                                 [ -d ~/.ssh ] || mkdir ~/.ssh && chmod 0700 ~/.ssh
                                 ssh-keyscan -t rsa,dsa github.com >> ~/.ssh/known_hosts
                             '''
-                                sh "go get -v -tags ${EE_BUILD_TAG} ./..."
+                                sh '.ci/retry.sh go mod download'
                             }
                         }
                     }

integration-test/e2e/Jenkinsfile

@@ -140,8 +140,11 @@ pipeline {
                         // both into FAILURE, so the unstable post block below would never run.
                         def exitCode = sh(returnStatus: true, script: '''
                             set -eu
-                            mkdir -p integration-test/e2e .github
+                            mkdir -p integration-test/e2e .github .ci
                             curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/e2e/run_e2e_tests.sh -o integration-test/e2e/run_e2e_tests.sh
+                            # run_e2e_tests.sh calls this, and SG_COMMIT may predate it, so take it from main too
+                            curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh
+                            chmod +x .ci/retry.sh
                             curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/slackUtils.groovy -o integration-test/slackUtils.groovy
                             curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.github/slack_usernames.yaml -o .github/slack_usernames.yaml
                             /bin/bash integration-test/e2e/run_e2e_tests.sh

integration-test/e2e/run_e2e_tests.sh

@@ -28,6 +28,9 @@ fi
 if [[ "$BACKING_STORE" == "cbs" ]]; then
     : "${COUCHBASE_SERVER_VERSION:?COUCHBASE_SERVER_VERSION must be set when BACKING_STORE=cbs}"
     CBS_ENV_FILE="$(mktemp)"
+    # start_cbs.py runs cbdinocluster out of the integration-test/tools module, so warm that
+    # module cache here where a flaky proxy is retried rather than failing the whole run
+    "${REPO_DIR}/.ci/retry.sh" go -C "${REPO_DIR}/integration-test/tools" mod download
     "${REPO_DIR}/integration-test/start_cbs.py" --version "${COUCHBASE_SERVER_VERSION}" --purpose sync_gateway_e2e --env-file "${CBS_ENV_FILE}"
     # shellcheck disable=SC1090
     source "${CBS_ENV_FILE}"
@@ -42,6 +45,10 @@ git config --global filter.lfs.clean "git-lfs clean -- %f"
 git config --global filter.lfs.smudge "git-lfs smudge -- %f"
 git config --global filter.lfs.process "git-lfs filter-process"
 
+# start_local.py builds Sync Gateway out of REPO_DIR further down, so warm the root module cache
+# now that the private module rewrite above is in place
+"${REPO_DIR}/.ci/retry.sh" go -C "${REPO_DIR}" mod download
+
 # Clean up any existing clone to make local re-runs idempotent
 rm -rf couchbase-lite-tests
 

jenkins-integration-build.sh

@@ -54,13 +54,19 @@ export GOPRIVATE=github.com/couchbaselabs/go-fleecedelta
 SG_COMMIT_HASH=$(git rev-parse HEAD)
 echo "Sync Gateway git commit hash: $SG_COMMIT_HASH"
 
-GO_VERSION=go$(go list -m -f '{{.GoVersion}}')
+GO_VERSION=go$(./.ci/retry.sh go list -m -f '{{.GoVersion}}')
 echo "Sync Gateway go.mod version is ${GO_VERSION}"
-go install "golang.org/dl/${GO_VERSION}@latest"
-~/go/bin/"${GO_VERSION}" download
+./.ci/retry.sh go install "golang.org/dl/${GO_VERSION}@latest"
+./.ci/retry.sh ~/go/bin/"${GO_VERSION}" download
 GOROOT=$(~/go/bin/"${GO_VERSION}" env GOROOT)
 PATH=${GOROOT}/bin:$PATH
 
+# Warm both module caches up front, so a flaky module proxy fails here where it is retried rather
+# than part-way through a test run. integration-test/tools is a separate module pinning
+# cbdinocluster, which start_cbs.py and the collect-logs call below both reach for.
+./.ci/retry.sh go mod download
+./.ci/retry.sh go -C integration-test/tools mod download
+
 if [ "${SG_TEST_X509:-}" == "true" ] && [ "${COUCHBASE_SERVER_PROTOCOL}" != "couchbases" ]; then
     echo "Setting SG_TEST_X509 requires using couchbases:// protocol, aborting integration tests"
     exit 1