Skip to content

ci: cleanup/re-organisation #8807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
torcolvin merged 2 commits into from
Sep 15, 2026
Merged

ci: cleanup/re-organisation #8807

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19a9f51
ci: move Jenkins job definitions and shared helpers into .ci
bbrks Sep 15, 2026
da7a6b8
ci: fetch the integration build script from main
bbrks Sep 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 20 additions & 14 deletions integration-test/MainIntegration/Jenkinsfile → .ci/jenkins/MainIntegration/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ def slackDetails(helpers) {
// post block, which would leave the failure unreported entirely.
def notifySlack(String status) {
def color = ['passed': 'good', 'aborted': 'warning'].get(status, 'danger')
if (!fileExists('integration-test/jenkinsHelpers.groovy')) {
if (!fileExists('.ci/jenkinsHelpers.groovy')) {
slackSend(channel: 'syncgatewaybot', color: color, message: ":x: *MainIntegration ${status}* - ${currentBuild.fullDisplayName}\n${env.BUILD_URL}")
return
}
def helpers = load('integration-test/jenkinsHelpers.groovy')
def helpers = load('.ci/jenkinsHelpers.groovy')
if (status == 'aborted' && helpers.isSelfCancelledAdhocBuild()) {
echo('Build was started and cancelled by the same user - skipping Slack notification')
return
Expand All @@ -44,11 +44,11 @@ def notifySlack(String status) {
// failed before that point - skip rather than throwing out of a post block, which would leave the result
// unreported entirely.
def notifyGitHub(String status) {
if (!fileExists('integration-test/jenkinsHelpers.groovy')) {
echo('integration-test/jenkinsHelpers.groovy not present - skipping GitHub commit status')
if (!fileExists('.ci/jenkinsHelpers.groovy')) {
echo('.ci/jenkinsHelpers.groovy not present - skipping GitHub commit status')
return
}
load('integration-test/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-main-integration-test', 'Integration tests', status)
load('.ci/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-main-integration-test', 'Integration tests', status)
}

pipeline {
Expand All @@ -62,7 +62,7 @@ pipeline {
githubProjectProperty(projectUrlStr: 'https://github.com/couchbase/sync_gateway/')
}

// Only the two parameters the run actually honours. The -m block in jenkins-integration-build.sh
// Only the two parameters the run actually honours. The -m block in .ci/jenkins/integration-build.sh
// hardcodes the server version, edition, target test and package, GSI and TLS settings, overwriting
// whatever this job sets - change those there, not here, and don't report them in Slack from here.
parameters {
Expand All @@ -78,7 +78,7 @@ pipeline {
// The `git` step only returns GIT_COMMIT rather than populating env, so assign it
// manually for the Slack commit link below. It also has no refspec option, so
// SG_COMMIT must name a branch or tag - a bare SHA won't resolve. Fine for this job;
// integration-test/e2e/Jenkinsfile has the checkout+refspec form if that changes.
// .ci/jenkins/e2e/Jenkinsfile has the checkout+refspec form if that changes.
def scmVars = git(
url: 'git@github.com:couchbase/sync_gateway.git',
branch: "${params.SG_COMMIT}",
Expand All @@ -87,18 +87,24 @@ pipeline {
env.GIT_COMMIT = scmVars.GIT_COMMIT
}
// SG_COMMIT can in principle be overridden to an arbitrary branch, whose copies of these
// helpers may be stale or missing methods the steps below need - always load the current
// ones from main instead.
// helpers and of the test runner may be stale or missing what the steps below need -
// always fetch the current ones from main instead.
sh '''
set -eu
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/jenkinsHelpers.groovy -o integration-test/jenkinsHelpers.groovy
mkdir -p .ci/jenkins
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkinsHelpers.groovy -o .ci/jenkinsHelpers.groovy
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/slack_usernames.yaml -o .ci/slack_usernames.yaml
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkins/integration-build.sh -o .ci/jenkins/integration-build.sh
# integration-build.sh calls this, and SG_COMMIT may predate it, so take it from main too
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh
chmod +x .ci/jenkins/integration-build.sh .ci/retry.sh
'''
script {
// Posted after the fetch above, and after the checkout since the status needs a SHA.
notifyGitHub('running')
// The build history lists the build number alone, so name what identifies a run
// here, the same way SyncGatewayIntegration/Jenkinsfile does.
def trigger = load('integration-test/jenkinsHelpers.groovy').triggeredBy()
def trigger = load('.ci/jenkinsHelpers.groovy').triggeredBy()
currentBuild.description = [params.SG_COMMIT, trigger ? "for ${trigger}" : ''].findAll { it }.join(' ')
}
}
Expand All @@ -107,14 +113,14 @@ pipeline {
steps {
sshagent(credentials: ['CB_SG_Robot_Github_SSH_Key']) {
script {
// jenkins-integration-build.sh separates test failures (exit 50 -> UNSTABLE) from a
// .ci/jenkins/integration-build.sh separates test failures (exit 50 -> UNSTABLE) from a
// test setup failure or unrecovered panic (exit 1 -> FAILURE). A plain sh step would
// collapse both into FAILURE, so the unstable post block below would never run.
def exitCode = sh(script: './jenkins-integration-build.sh -m', returnStatus: true)
def exitCode = sh(script: './.ci/jenkins/integration-build.sh -m', returnStatus: true)
if (exitCode == 50) {
unstable('At least one integration test failed')
} else if (exitCode != 0) {
error("jenkins-integration-build.sh exited ${exitCode}")
error(".ci/jenkins/integration-build.sh exited ${exitCode}")
}
}
}
Expand Down
32 changes: 19 additions & 13 deletions ...n-test/SyncGatewayIntegration/Jenkinsfile → ...enkins/SyncGatewayIntegration/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

// Note for whoever creates this job: its *first* build runs with no parameter values at all. Jenkins
// registers the parameters{} block when it first evaluates this file, but only applies the defaults from the
// second build onwards, so that first run dies in jenkins-integration-build.sh's REQUIRED_VARS check with
// second build onwards, so that first run dies in .ci/jenkins/integration-build.sh's REQUIRED_VARS check with
// "COUCHBASE_SERVER_PROTOCOL environment variable is required to be set". Just run it again - it isn't a
// misconfiguration.

Expand All @@ -26,11 +26,11 @@ def slackDetails(helpers) {
// post block, which would leave the failure unreported entirely.
def notifySlack(String status) {
def color = ['passed': 'good', 'aborted': 'warning'].get(status, 'danger')
if (!fileExists('integration-test/jenkinsHelpers.groovy')) {
if (!fileExists('.ci/jenkinsHelpers.groovy')) {
slackSend(channel: 'syncgatewaybot', color: color, message: ":x: *SyncGatewayIntegration ${status}* - ${currentBuild.fullDisplayName}\n${env.BUILD_URL}")
return
}
def helpers = load('integration-test/jenkinsHelpers.groovy')
def helpers = load('.ci/jenkinsHelpers.groovy')
if (status == 'aborted' && helpers.isSelfCancelledAdhocBuild()) {
echo('Build was started and cancelled by the same user - skipping Slack notification')
return
Expand All @@ -49,11 +49,11 @@ def notifySlack(String status) {
// failed before that point - skip rather than throwing out of a post block, which would leave the result
// unreported entirely.
def notifyGitHub(String status) {
if (!fileExists('integration-test/jenkinsHelpers.groovy')) {
echo('integration-test/jenkinsHelpers.groovy not present - skipping GitHub commit status')
if (!fileExists('.ci/jenkinsHelpers.groovy')) {
echo('.ci/jenkinsHelpers.groovy not present - skipping GitHub commit status')
return
}
load('integration-test/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-integration-test', 'Integration tests', status)
load('.ci/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-integration-test', 'Integration tests', status)
}

pipeline {
Expand Down Expand Up @@ -95,7 +95,7 @@ To use pre-release versions, find the number on https://github.com/orgs/cb-vanil
)
string(name: 'SG_TEST_BUCKET_POOL_SIZE', defaultValue: '4', description: 'Number of buckets to create in by the test harness.')
string(name: 'RUN_COUNT', defaultValue: '', description: 'Adds -count <n> parameter to test runs. Default is not specified.')
// jenkins-integration-build.sh reads RUN_WALRUS unconditionally under `set -u`, so it has to be set.
// .ci/jenkins/integration-build.sh reads RUN_WALRUS unconditionally under `set -u`, so it has to be set.
// Declaring it here is enough: Jenkins exposes parameters as environment variables.
booleanParam(name: 'RUN_WALRUS', defaultValue: false, description: 'Run rosmar tests in addition to the Couchbase Server tests.')
booleanParam(name: 'SG_TEST_BUCKET_POOL_DEBUG', defaultValue: false, description: 'This will log the test bucket pool creation and cleanup.')
Expand Down Expand Up @@ -143,12 +143,18 @@ To use pre-release versions, find the number on https://github.com/orgs/cb-vanil
env.GIT_COMMIT = scmVars.GIT_COMMIT
env.GIT_BRANCH = scmVars.GIT_BRANCH
}
// SG_COMMIT can be an arbitrary developer branch, whose copies of these helpers may be
// stale or missing methods the steps below need - always load the current ones from main
// instead.
// SG_COMMIT can be an arbitrary developer branch, whose copies of these helpers and of
// the test runner may be stale or missing what the steps below need - always fetch the
// current ones from main instead.
sh '''
set -eu
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/jenkinsHelpers.groovy -o integration-test/jenkinsHelpers.groovy
mkdir -p .ci/jenkins
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkinsHelpers.groovy -o .ci/jenkinsHelpers.groovy
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/slack_usernames.yaml -o .ci/slack_usernames.yaml
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkins/integration-build.sh -o .ci/jenkins/integration-build.sh
# integration-build.sh calls this, and SG_COMMIT may predate it, so take it from main too
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh
chmod +x .ci/jenkins/integration-build.sh .ci/retry.sh
'''
script {
// Posted after the fetch above, and after the checkout since the status needs a SHA.
Expand All @@ -170,7 +176,7 @@ To use pre-release versions, find the number on https://github.com/orgs/cb-vanil
sshagent(credentials: ['CB_SG_Robot_Github_SSH_Key']) {
withCredentials([usernamePassword(credentialsId: 'github_ghcr_pat', usernameVariable: 'GHCR_USER', passwordVariable: 'GHCR_TOKEN')]) {
script {
// `set -e` propagates jenkins-integration-build.sh's exit status, which separates
// `set -e` propagates .ci/jenkins/integration-build.sh's exit status, which separates
// test failures (50 -> UNSTABLE) from a test setup failure or unrecovered panic
// (1 -> FAILURE). A plain sh step would collapse both into FAILURE, so the
// unstable post block below would never run.
Expand All @@ -183,7 +189,7 @@ To use pre-release versions, find the number on https://github.com/orgs/cb-vanil
IMAGE="couchbase/server:${COUCHBASE_SERVER_VERSION}"
fi
docker pull "$IMAGE"
./jenkins-integration-build.sh
./.ci/jenkins/integration-build.sh
Comment thread
bbrks marked this conversation as resolved.
''')
if (exitCode == 50) {
unstable('At least one integration test failed')
Expand Down
0 .../SyncGatewayIntegrationWeekly/Jenkinsfile → .../SyncGatewayIntegrationWeekly/Jenkinsfile
View file
Open in desktop
File renamed without changes.
38 changes: 22 additions & 16 deletions integration-test/TopologyTests/Jenkinsfile → .ci/jenkins/TopologyTests/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

// Note for whoever creates this job: its *first* build runs with no parameter values at all. Jenkins
// registers the parameters{} block when it first evaluates this file, but only applies the defaults from the
// second build onwards, so that first run dies in jenkins-integration-build.sh's REQUIRED_VARS check with
// second build onwards, so that first run dies in .ci/jenkins/integration-build.sh's REQUIRED_VARS check with
// "COUCHBASE_SERVER_PROTOCOL environment variable is required to be set". Just run it again - it isn't a
// misconfiguration.

Expand All @@ -27,11 +27,11 @@ def slackDetails(helpers) {
// post block, which would leave the failure unreported entirely.
def notifySlack(String status) {
def color = ['passed': 'good', 'aborted': 'warning'].get(status, 'danger')
if (!fileExists('integration-test/jenkinsHelpers.groovy')) {
if (!fileExists('.ci/jenkinsHelpers.groovy')) {
slackSend(channel: 'syncgatewaybot', color: color, message: ":x: *TopologyTests ${status}* - ${currentBuild.fullDisplayName}\n${env.BUILD_URL}")
return
}
def helpers = load('integration-test/jenkinsHelpers.groovy')
def helpers = load('.ci/jenkinsHelpers.groovy')
if (status == 'aborted' && helpers.isSelfCancelledAdhocBuild()) {
echo('Build was started and cancelled by the same user - skipping Slack notification')
return
Expand All @@ -50,11 +50,11 @@ def notifySlack(String status) {
// failed before that point - skip rather than throwing out of a post block, which would leave the result
// unreported entirely.
def notifyGitHub(String status) {
if (!fileExists('integration-test/jenkinsHelpers.groovy')) {
echo('integration-test/jenkinsHelpers.groovy not present - skipping GitHub commit status')
if (!fileExists('.ci/jenkinsHelpers.groovy')) {
echo('.ci/jenkinsHelpers.groovy not present - skipping GitHub commit status')
return
}
load('integration-test/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-topology-tests', 'Topology tests', status)
load('.ci/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-topology-tests', 'Topology tests', status)
}

pipeline {
Expand Down Expand Up @@ -87,7 +87,7 @@ pipeline {
}

environment {
// jenkins-integration-build.sh reads RUN_WALRUS unconditionally under `set -u`, so it has to be
// .ci/jenkins/integration-build.sh reads RUN_WALRUS unconditionally under `set -u`, so it has to be
// set even though this job never runs the Rosmar leg.
RUN_WALRUS = 'false'
TARGET_PACKAGE = 'topologytest'
Expand All @@ -102,27 +102,33 @@ pipeline {
// The `git` step only returns GIT_COMMIT rather than populating env, so assign it
// manually for the Slack commit link below. It also has no refspec option, so
// SG_COMMIT must name a branch or tag - a bare SHA won't resolve. Fine for this job;
// integration-test/e2e/Jenkinsfile has the checkout+refspec form if that changes.
// .ci/jenkins/e2e/Jenkinsfile has the checkout+refspec form if that changes.
def scmVars = git(
url: 'git@github.com:couchbase/sync_gateway.git',
branch: "${params.SG_COMMIT}",
credentialsId: 'CB_SG_Robot_Github_SSH_Key'
)
env.GIT_COMMIT = scmVars.GIT_COMMIT
}
// SG_COMMIT can be an arbitrary developer branch, whose copies of these helpers may be
// stale or missing methods the steps below need - always load the current ones from main
// instead.
// SG_COMMIT can be an arbitrary developer branch, whose copies of these helpers and of
// the test runner may be stale or missing what the steps below need - always fetch the
// current ones from main instead.
sh '''
set -eu
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/jenkinsHelpers.groovy -o integration-test/jenkinsHelpers.groovy
mkdir -p .ci/jenkins
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkinsHelpers.groovy -o .ci/jenkinsHelpers.groovy
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/slack_usernames.yaml -o .ci/slack_usernames.yaml
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkins/integration-build.sh -o .ci/jenkins/integration-build.sh
# integration-build.sh calls this, and SG_COMMIT may predate it, so take it from main too
curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh
chmod +x .ci/jenkins/integration-build.sh .ci/retry.sh
'''
script {
// Posted after the fetch above, and after the checkout since the status needs a SHA.
notifyGitHub('running')
// The build history lists the build number alone, so name the parameters that
// identify a run here, the same way SyncGatewayIntegration/Jenkinsfile does.
def trigger = load('integration-test/jenkinsHelpers.groovy').triggeredBy()
def trigger = load('.ci/jenkinsHelpers.groovy').triggeredBy()
def targetTest = params.TARGET_TEST == 'ALL' ? '' : params.TARGET_TEST
currentBuild.description = [
params.SG_COMMIT,
Expand All @@ -137,14 +143,14 @@ pipeline {
steps {
sshagent(credentials: ['CB_SG_Robot_Github_SSH_Key']) {
script {
// jenkins-integration-build.sh separates test failures (exit 50 -> UNSTABLE) from a
// .ci/jenkins/integration-build.sh separates test failures (exit 50 -> UNSTABLE) from a
// test setup failure or unrecovered panic (exit 1 -> FAILURE). A plain sh step would
// collapse both into FAILURE, so the unstable post block below would never run.
def exitCode = sh(script: './jenkins-integration-build.sh', returnStatus: true)
def exitCode = sh(script: './.ci/jenkins/integration-build.sh', returnStatus: true)
if (exitCode == 50) {
unstable('At least one topology test failed')
} else if (exitCode != 0) {
error("jenkins-integration-build.sh exited ${exitCode}")
error(".ci/jenkins/integration-build.sh exited ${exitCode}")
}
}
}
Expand Down
Loading
Loading