From 17b5691f5e64b78bd8218c8a72de364ffe156624 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Mon, 14 Sep 2026 12:39:24 +0100 Subject: [PATCH 1/3] CBG-5862: retry Go module downloads in CI The module proxy has been returning transient errors for weeks, failing builds in a way that says nothing about the code under test. Add .ci/retry.sh and give every CI job an explicit, retried download step before it builds or tests. Builds and tests stay unwrapped: the cache is warm by then, and retrying a test run would mask genuine flakes. GitHub Actions warms the cache with a package-scoped go list rather than go mod download. go-fleecedelta is a direct requirement but private, and go mod download ignores build tags, so it fails on runners without the EE credentials. The tool meta-pattern covers gotestsum and addlicense, which nothing in the tree imports. Jenkins has the credentials, so it uses go mod download, which needs no tags and covers the CE leg, the EE leg and the tool modules at once. Jenkins also warms integration-test/tools, the separate module pinning cbdinocluster, which start_cbs.py reaches for. Co-Authored-By: Claude Opus 5 (1M context) --- .ci/retry.sh | 38 +++++++++++++++++++++++++ .github/workflows/ci.yml | 40 +++++++++++++++++++++++++++ .github/workflows/service.yml | 4 +++ .github/workflows/shell.yml | 2 ++ Jenkinsfile | 8 +++--- integration-test/e2e/run_e2e_tests.sh | 3 ++ jenkins-integration-build.sh | 12 ++++++-- 7 files changed, 100 insertions(+), 7 deletions(-) create mode 100755 .ci/retry.sh diff --git a/.ci/retry.sh b/.ci/retry.sh new file mode 100755 index 0000000000..fed3bac7b5 --- /dev/null +++ b/.ci/retry.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Copyright 2026-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included in +# the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +# file, in accordance with the Business Source License, use of this software +# will be governed by the Apache License, Version 2.0, included in the file +# licenses/APL2.txt. + +# Runs a command, retrying with exponential backoff. CI uses this for Go module downloads, which +# fail intermittently when the module proxy is unhealthy. +# +# Usage: .ci/retry.sh [args...] +# Tunable with RETRY_ATTEMPTS (default 4) and RETRY_DELAY (initial backoff seconds, default 5). + +set -euo pipefail + +attempts=${RETRY_ATTEMPTS:-4} +delay=${RETRY_DELAY:-5} + +for ((attempt = 1; attempt <= attempts; attempt++)); do + # capture the status here: $? after an if-block is the status of the if, not of the command + status=0 + "$@" || status=$? + if ((status == 0)); then + exit 0 + fi + if ((attempt < attempts)); then + # diagnostics go to stderr so callers capturing stdout get the command's output only + echo "retry: attempt ${attempt}/${attempts} of '$*' failed (exit ${status}), retrying in ${delay}s" >&2 + sleep "${delay}" + delay=$((delay * 2)) + fi +done + +echo "retry: '$*' failed after ${attempts} attempts" >&2 +exit "${status}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5de66e559a..8de8bf15df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: go-build run: go build "./..." @@ -63,6 +67,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - run: go tool addlicense -check -f licenses/addlicense.tmpl . golangci: @@ -73,6 +81,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: golangci-lint uses: *actions_golangci_lint_version with: @@ -100,6 +112,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: Build run: go build -v "./..." - name: Run Tests @@ -136,6 +152,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -race -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -171,6 +191,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -204,6 +228,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 -short -bench=. -benchtime=1x -run '!' "./..." shell: bash @@ -240,6 +268,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -304,6 +336,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: Build run: go build -v "./tools/stats-definition-exporter" - name: Run Tests @@ -338,6 +374,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: Build run: go build "./tools/cache_perf_tool" govulncheck: diff --git a/.github/workflows/service.yml b/.github/workflows/service.yml index ae009827f2..b72158ce88 100644 --- a/.github/workflows/service.yml +++ b/.github/workflows/service.yml @@ -53,6 +53,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: go.mod + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null - name: "Build Sync Gateway" run: mkdir -p ./bin && go build -o ./bin ./... - name: "Run test scripts" diff --git a/.github/workflows/shell.yml b/.github/workflows/shell.yml index 898ea1f3c6..20442aca04 100644 --- a/.github/workflows/shell.yml +++ b/.github/workflows/shell.yml @@ -15,6 +15,7 @@ on: # Only run when we change a shell script paths: - "*.sh" + - ".ci/*.sh" branches: - "main" - "release/*" @@ -25,6 +26,7 @@ on: # Only run when we change a shell script paths: - "*.sh" + - ".ci/*.sh" branches: - "main" - "release/*" diff --git a/Jenkinsfile b/Jenkinsfile index 9cd52685de..cc68a36333 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,7 +38,7 @@ pipeline { env.GO_VERSION = 'go' + sh( returnStdout: true, script: ''' - go list -m -f '{{.GoVersion}}' + .ci/retry.sh go list -m -f '{{.GoVersion}}' ''' ).trim() // go fetches the toolchain named in go.mod on demand @@ -46,8 +46,8 @@ pipeline { set -eux echo "Sync Gateway go.mod version is $GO_VERSION" - go install "golang.org/dl/$GO_VERSION@latest" - ~/go/bin/$GO_VERSION download + .ci/retry.sh go install "golang.org/dl/$GO_VERSION@latest" + .ci/retry.sh ~/go/bin/$GO_VERSION download ''' env.GOROOT = sh( returnStdout: true, @@ -68,7 +68,7 @@ pipeline { [ -d ~/.ssh ] || mkdir ~/.ssh && chmod 0700 ~/.ssh ssh-keyscan -t rsa,dsa github.com >> ~/.ssh/known_hosts ''' - sh "go get -v -tags ${EE_BUILD_TAG} ./..." + sh '.ci/retry.sh go mod download' } } } diff --git a/integration-test/e2e/run_e2e_tests.sh b/integration-test/e2e/run_e2e_tests.sh index c28167819b..5fba6d1cff 100755 --- a/integration-test/e2e/run_e2e_tests.sh +++ b/integration-test/e2e/run_e2e_tests.sh @@ -28,6 +28,9 @@ fi if [[ "$BACKING_STORE" == "cbs" ]]; then : "${COUCHBASE_SERVER_VERSION:?COUCHBASE_SERVER_VERSION must be set when BACKING_STORE=cbs}" CBS_ENV_FILE="$(mktemp)" + # start_cbs.py runs cbdinocluster out of the integration-test/tools module, so warm that + # module cache here where a flaky proxy is retried rather than failing the whole run + "${REPO_DIR}/.ci/retry.sh" go -C "${REPO_DIR}/integration-test/tools" mod download "${REPO_DIR}/integration-test/start_cbs.py" --version "${COUCHBASE_SERVER_VERSION}" --purpose sync_gateway_e2e --env-file "${CBS_ENV_FILE}" # shellcheck disable=SC1090 source "${CBS_ENV_FILE}" diff --git a/jenkins-integration-build.sh b/jenkins-integration-build.sh index 6067d74321..dc51ac7882 100755 --- a/jenkins-integration-build.sh +++ b/jenkins-integration-build.sh @@ -54,13 +54,19 @@ export GOPRIVATE=github.com/couchbaselabs/go-fleecedelta SG_COMMIT_HASH=$(git rev-parse HEAD) echo "Sync Gateway git commit hash: $SG_COMMIT_HASH" -GO_VERSION=go$(go list -m -f '{{.GoVersion}}') +GO_VERSION=go$(./.ci/retry.sh go list -m -f '{{.GoVersion}}') echo "Sync Gateway go.mod version is ${GO_VERSION}" -go install "golang.org/dl/${GO_VERSION}@latest" -~/go/bin/"${GO_VERSION}" download +./.ci/retry.sh go install "golang.org/dl/${GO_VERSION}@latest" +./.ci/retry.sh ~/go/bin/"${GO_VERSION}" download GOROOT=$(~/go/bin/"${GO_VERSION}" env GOROOT) PATH=${GOROOT}/bin:$PATH +# Warm both module caches up front, so a flaky module proxy fails here where it is retried rather +# than part-way through a test run. integration-test/tools is a separate module pinning +# cbdinocluster, which start_cbs.py and the collect-logs call below both reach for. +./.ci/retry.sh go mod download +./.ci/retry.sh go -C integration-test/tools mod download + if [ "${SG_TEST_X509:-}" == "true" ] && [ "${COUCHBASE_SERVER_PROTOCOL}" != "couchbases" ]; then echo "Setting SG_TEST_X509 requires using couchbases:// protocol, aborting integration tests" exit 1 From 90d861e04950fad11793a4913d9c833ffa5ec65c Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Mon, 14 Sep 2026 12:57:40 +0100 Subject: [PATCH 2/3] CBG-5862: validate retry.sh inputs and fix the e2e helper path retry.sh exited 0 without running anything when called with no arguments, and died on an unbound status when RETRY_ATTEMPTS was zero or non-numeric. Reject both up front, along with a non-numeric RETRY_DELAY. The e2e pipeline overwrites run_e2e_tests.sh with main's copy even when it checks out an arbitrary SG_COMMIT, so the helper has to come from main too or every CBS-backed run against an older commit dies on a missing file. curl drops the exec bit, hence the chmod. Warm the root module cache in run_e2e_tests.sh as well: start_local.py builds Sync Gateway out of REPO_DIR, which a cold worker would otherwise download unretried. It goes after the private module rewrite so go-fleecedelta still resolves. Co-Authored-By: Claude Opus 5 (1M context) --- .ci/retry.sh | 14 ++++++++++++++ integration-test/e2e/Jenkinsfile | 5 ++++- integration-test/e2e/run_e2e_tests.sh | 4 ++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.ci/retry.sh b/.ci/retry.sh index fed3bac7b5..b2de99882e 100755 --- a/.ci/retry.sh +++ b/.ci/retry.sh @@ -19,6 +19,20 @@ set -euo pipefail attempts=${RETRY_ATTEMPTS:-4} delay=${RETRY_DELAY:-5} +# without these guards, no args exits 0 without running anything and a bad attempt count skips the loop entirely +if (($# == 0)); then + echo "retry: usage: .ci/retry.sh [args...]" >&2 + exit 2 +fi +if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then + echo "retry: RETRY_ATTEMPTS must be a positive integer, got '${attempts}'" >&2 + exit 2 +fi +if [[ ! "${delay}" =~ ^[0-9]+$ ]]; then + echo "retry: RETRY_DELAY must be a non-negative integer, got '${delay}'" >&2 + exit 2 +fi + for ((attempt = 1; attempt <= attempts; attempt++)); do # capture the status here: $? after an if-block is the status of the if, not of the command status=0 diff --git a/integration-test/e2e/Jenkinsfile b/integration-test/e2e/Jenkinsfile index de3fd1780d..6f7d8e7dba 100644 --- a/integration-test/e2e/Jenkinsfile +++ b/integration-test/e2e/Jenkinsfile @@ -136,8 +136,11 @@ pipeline { sshagent(credentials: ['CB_SG_Robot_Github_SSH_Key']) { sh ''' set -eu - mkdir -p integration-test/e2e .github + mkdir -p integration-test/e2e .github .ci curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/e2e/run_e2e_tests.sh -o integration-test/e2e/run_e2e_tests.sh + # run_e2e_tests.sh calls this, and SG_COMMIT may predate it, so take it from main too + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh + chmod +x .ci/retry.sh curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/slackUtils.groovy -o integration-test/slackUtils.groovy curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.github/slack_usernames.yaml -o .github/slack_usernames.yaml /bin/bash integration-test/e2e/run_e2e_tests.sh diff --git a/integration-test/e2e/run_e2e_tests.sh b/integration-test/e2e/run_e2e_tests.sh index 5fba6d1cff..566b815326 100755 --- a/integration-test/e2e/run_e2e_tests.sh +++ b/integration-test/e2e/run_e2e_tests.sh @@ -45,6 +45,10 @@ git config --global filter.lfs.clean "git-lfs clean -- %f" git config --global filter.lfs.smudge "git-lfs smudge -- %f" git config --global filter.lfs.process "git-lfs filter-process" +# start_local.py builds Sync Gateway out of REPO_DIR further down, so warm the root module cache +# now that the private module rewrite above is in place +"${REPO_DIR}/.ci/retry.sh" go -C "${REPO_DIR}" mod download + # Clean up any existing clone to make local re-runs idempotent rm -rf couchbase-lite-tests From 8bb8d25c46dae514a7dcad96de9e495e1192907d Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Mon, 14 Sep 2026 15:48:41 +0100 Subject: [PATCH 3/3] CBG-5862: prefetch the ruleguard module deps too ruleguard/ only builds under the ruleguard tag, so the dependency warm-up skipped it and golangci-lint had to fetch go-ruleguard/dsl mid-lint, outside the retry wrapper. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 20 ++++++++++---------- .github/workflows/service.yml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8de8bf15df..3e5a704197 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: go-build run: go build "./..." @@ -70,7 +70,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - run: go tool addlicense -check -f licenses/addlicense.tmpl . golangci: @@ -84,7 +84,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: golangci-lint uses: *actions_golangci_lint_version with: @@ -115,7 +115,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Build run: go build -v "./..." - name: Run Tests @@ -155,7 +155,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -race -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -194,7 +194,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -231,7 +231,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 -short -bench=. -benchtime=1x -run '!' "./..." shell: bash @@ -271,7 +271,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -339,7 +339,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Build run: go build -v "./tools/stats-definition-exporter" - name: Run Tests @@ -377,7 +377,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Build run: go build "./tools/cache_perf_tool" govulncheck: diff --git a/.github/workflows/service.yml b/.github/workflows/service.yml index b72158ce88..8c2d8c8f1c 100644 --- a/.github/workflows/service.yml +++ b/.github/workflows/service.yml @@ -56,7 +56,7 @@ jobs: # go mod download would try to fetch the private EE module, which public runners can't reach - name: Download Go modules shell: bash - run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode ./... tool > /dev/null + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: "Build Sync Gateway" run: mkdir -p ./bin && go build -o ./bin ./... - name: "Run test scripts"