diff --git a/.ci/retry.sh b/.ci/retry.sh new file mode 100755 index 0000000000..b2de99882e --- /dev/null +++ b/.ci/retry.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Copyright 2026-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included in +# the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +# file, in accordance with the Business Source License, use of this software +# will be governed by the Apache License, Version 2.0, included in the file +# licenses/APL2.txt. + +# Runs a command, retrying with exponential backoff. CI uses this for Go module downloads, which +# fail intermittently when the module proxy is unhealthy. +# +# Usage: .ci/retry.sh [args...] +# Tunable with RETRY_ATTEMPTS (default 4) and RETRY_DELAY (initial backoff seconds, default 5). + +set -euo pipefail + +attempts=${RETRY_ATTEMPTS:-4} +delay=${RETRY_DELAY:-5} + +# without these guards, no args exits 0 without running anything and a bad attempt count skips the loop entirely +if (($# == 0)); then + echo "retry: usage: .ci/retry.sh [args...]" >&2 + exit 2 +fi +if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then + echo "retry: RETRY_ATTEMPTS must be a positive integer, got '${attempts}'" >&2 + exit 2 +fi +if [[ ! "${delay}" =~ ^[0-9]+$ ]]; then + echo "retry: RETRY_DELAY must be a non-negative integer, got '${delay}'" >&2 + exit 2 +fi + +for ((attempt = 1; attempt <= attempts; attempt++)); do + # capture the status here: $? after an if-block is the status of the if, not of the command + status=0 + "$@" || status=$? + if ((status == 0)); then + exit 0 + fi + if ((attempt < attempts)); then + # diagnostics go to stderr so callers capturing stdout get the command's output only + echo "retry: attempt ${attempt}/${attempts} of '$*' failed (exit ${status}), retrying in ${delay}s" >&2 + sleep "${delay}" + delay=$((delay * 2)) + fi +done + +echo "retry: '$*' failed after ${attempts} attempts" >&2 +exit "${status}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5de66e559a..3e5a704197 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: go-build run: go build "./..." @@ -63,6 +67,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - run: go tool addlicense -check -f licenses/addlicense.tmpl . golangci: @@ -73,6 +81,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: golangci-lint uses: *actions_golangci_lint_version with: @@ -100,6 +112,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Build run: go build -v "./..." - name: Run Tests @@ -136,6 +152,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -race -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -171,6 +191,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -204,6 +228,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 -short -bench=. -benchtime=1x -run '!' "./..." shell: bash @@ -240,6 +268,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Run Tests run: go tool gotestsum --format pkgname-and-test-fails --hide-summary=skipped --hide-summary=output --jsonfile test.json -- -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 "./..." shell: bash @@ -304,6 +336,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Build run: go build -v "./tools/stats-definition-exporter" - name: Run Tests @@ -338,6 +374,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: *go_mod_file + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: Build run: go build "./tools/cache_perf_tool" govulncheck: diff --git a/.github/workflows/service.yml b/.github/workflows/service.yml index ae009827f2..8c2d8c8f1c 100644 --- a/.github/workflows/service.yml +++ b/.github/workflows/service.yml @@ -53,6 +53,10 @@ jobs: - uses: *actions_setup_go_version with: go-version-file: go.mod + # go mod download would try to fetch the private EE module, which public runners can't reach + - name: Download Go modules + shell: bash + run: .ci/retry.sh go list -deps -test -tags cb_sg_devmode,ruleguard ./... tool > /dev/null - name: "Build Sync Gateway" run: mkdir -p ./bin && go build -o ./bin ./... - name: "Run test scripts" diff --git a/.github/workflows/shell.yml b/.github/workflows/shell.yml index 898ea1f3c6..20442aca04 100644 --- a/.github/workflows/shell.yml +++ b/.github/workflows/shell.yml @@ -15,6 +15,7 @@ on: # Only run when we change a shell script paths: - "*.sh" + - ".ci/*.sh" branches: - "main" - "release/*" @@ -25,6 +26,7 @@ on: # Only run when we change a shell script paths: - "*.sh" + - ".ci/*.sh" branches: - "main" - "release/*" diff --git a/Jenkinsfile b/Jenkinsfile index 9cd52685de..cc68a36333 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,7 +38,7 @@ pipeline { env.GO_VERSION = 'go' + sh( returnStdout: true, script: ''' - go list -m -f '{{.GoVersion}}' + .ci/retry.sh go list -m -f '{{.GoVersion}}' ''' ).trim() // go fetches the toolchain named in go.mod on demand @@ -46,8 +46,8 @@ pipeline { set -eux echo "Sync Gateway go.mod version is $GO_VERSION" - go install "golang.org/dl/$GO_VERSION@latest" - ~/go/bin/$GO_VERSION download + .ci/retry.sh go install "golang.org/dl/$GO_VERSION@latest" + .ci/retry.sh ~/go/bin/$GO_VERSION download ''' env.GOROOT = sh( returnStdout: true, @@ -68,7 +68,7 @@ pipeline { [ -d ~/.ssh ] || mkdir ~/.ssh && chmod 0700 ~/.ssh ssh-keyscan -t rsa,dsa github.com >> ~/.ssh/known_hosts ''' - sh "go get -v -tags ${EE_BUILD_TAG} ./..." + sh '.ci/retry.sh go mod download' } } } diff --git a/integration-test/e2e/Jenkinsfile b/integration-test/e2e/Jenkinsfile index 3e09b89003..6b00631cbe 100644 --- a/integration-test/e2e/Jenkinsfile +++ b/integration-test/e2e/Jenkinsfile @@ -140,8 +140,11 @@ pipeline { // both into FAILURE, so the unstable post block below would never run. def exitCode = sh(returnStatus: true, script: ''' set -eu - mkdir -p integration-test/e2e .github + mkdir -p integration-test/e2e .github .ci curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/e2e/run_e2e_tests.sh -o integration-test/e2e/run_e2e_tests.sh + # run_e2e_tests.sh calls this, and SG_COMMIT may predate it, so take it from main too + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh + chmod +x .ci/retry.sh curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/slackUtils.groovy -o integration-test/slackUtils.groovy curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.github/slack_usernames.yaml -o .github/slack_usernames.yaml /bin/bash integration-test/e2e/run_e2e_tests.sh diff --git a/integration-test/e2e/run_e2e_tests.sh b/integration-test/e2e/run_e2e_tests.sh index 787a12a3ea..e8cb99a9f2 100755 --- a/integration-test/e2e/run_e2e_tests.sh +++ b/integration-test/e2e/run_e2e_tests.sh @@ -28,6 +28,9 @@ fi if [[ "$BACKING_STORE" == "cbs" ]]; then : "${COUCHBASE_SERVER_VERSION:?COUCHBASE_SERVER_VERSION must be set when BACKING_STORE=cbs}" CBS_ENV_FILE="$(mktemp)" + # start_cbs.py runs cbdinocluster out of the integration-test/tools module, so warm that + # module cache here where a flaky proxy is retried rather than failing the whole run + "${REPO_DIR}/.ci/retry.sh" go -C "${REPO_DIR}/integration-test/tools" mod download "${REPO_DIR}/integration-test/start_cbs.py" --version "${COUCHBASE_SERVER_VERSION}" --purpose sync_gateway_e2e --env-file "${CBS_ENV_FILE}" # shellcheck disable=SC1090 source "${CBS_ENV_FILE}" @@ -42,6 +45,10 @@ git config --global filter.lfs.clean "git-lfs clean -- %f" git config --global filter.lfs.smudge "git-lfs smudge -- %f" git config --global filter.lfs.process "git-lfs filter-process" +# start_local.py builds Sync Gateway out of REPO_DIR further down, so warm the root module cache +# now that the private module rewrite above is in place +"${REPO_DIR}/.ci/retry.sh" go -C "${REPO_DIR}" mod download + # Clean up any existing clone to make local re-runs idempotent rm -rf couchbase-lite-tests diff --git a/jenkins-integration-build.sh b/jenkins-integration-build.sh index 6067d74321..dc51ac7882 100755 --- a/jenkins-integration-build.sh +++ b/jenkins-integration-build.sh @@ -54,13 +54,19 @@ export GOPRIVATE=github.com/couchbaselabs/go-fleecedelta SG_COMMIT_HASH=$(git rev-parse HEAD) echo "Sync Gateway git commit hash: $SG_COMMIT_HASH" -GO_VERSION=go$(go list -m -f '{{.GoVersion}}') +GO_VERSION=go$(./.ci/retry.sh go list -m -f '{{.GoVersion}}') echo "Sync Gateway go.mod version is ${GO_VERSION}" -go install "golang.org/dl/${GO_VERSION}@latest" -~/go/bin/"${GO_VERSION}" download +./.ci/retry.sh go install "golang.org/dl/${GO_VERSION}@latest" +./.ci/retry.sh ~/go/bin/"${GO_VERSION}" download GOROOT=$(~/go/bin/"${GO_VERSION}" env GOROOT) PATH=${GOROOT}/bin:$PATH +# Warm both module caches up front, so a flaky module proxy fails here where it is retried rather +# than part-way through a test run. integration-test/tools is a separate module pinning +# cbdinocluster, which start_cbs.py and the collect-logs call below both reach for. +./.ci/retry.sh go mod download +./.ci/retry.sh go -C integration-test/tools mod download + if [ "${SG_TEST_X509:-}" == "true" ] && [ "${COUCHBASE_SERVER_PROTOCOL}" != "couchbases" ]; then echo "Setting SG_TEST_X509 requires using couchbases:// protocol, aborting integration tests" exit 1