diff --git a/integration-test/MainIntegration/Jenkinsfile b/.ci/jenkins/MainIntegration/Jenkinsfile similarity index 78% rename from integration-test/MainIntegration/Jenkinsfile rename to .ci/jenkins/MainIntegration/Jenkinsfile index 67bbc2317a..496da15be0 100644 --- a/integration-test/MainIntegration/Jenkinsfile +++ b/.ci/jenkins/MainIntegration/Jenkinsfile @@ -21,11 +21,11 @@ def slackDetails(helpers) { // post block, which would leave the failure unreported entirely. def notifySlack(String status) { def color = ['passed': 'good', 'aborted': 'warning'].get(status, 'danger') - if (!fileExists('integration-test/jenkinsHelpers.groovy')) { + if (!fileExists('.ci/jenkinsHelpers.groovy')) { slackSend(channel: 'syncgatewaybot', color: color, message: ":x: *MainIntegration ${status}* - ${currentBuild.fullDisplayName}\n${env.BUILD_URL}") return } - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') if (status == 'aborted' && helpers.isSelfCancelledAdhocBuild()) { echo('Build was started and cancelled by the same user - skipping Slack notification') return @@ -44,11 +44,11 @@ def notifySlack(String status) { // failed before that point - skip rather than throwing out of a post block, which would leave the result // unreported entirely. def notifyGitHub(String status) { - if (!fileExists('integration-test/jenkinsHelpers.groovy')) { - echo('integration-test/jenkinsHelpers.groovy not present - skipping GitHub commit status') + if (!fileExists('.ci/jenkinsHelpers.groovy')) { + echo('.ci/jenkinsHelpers.groovy not present - skipping GitHub commit status') return } - load('integration-test/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-main-integration-test', 'Integration tests', status) + load('.ci/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-main-integration-test', 'Integration tests', status) } pipeline { @@ -62,7 +62,7 @@ pipeline { githubProjectProperty(projectUrlStr: 'https://github.com/couchbase/sync_gateway/') } - // Only the two parameters the run actually honours. The -m block in jenkins-integration-build.sh + // Only the two parameters the run actually honours. The -m block in .ci/jenkins/integration-build.sh // hardcodes the server version, edition, target test and package, GSI and TLS settings, overwriting // whatever this job sets - change those there, not here, and don't report them in Slack from here. parameters { @@ -78,7 +78,7 @@ pipeline { // The `git` step only returns GIT_COMMIT rather than populating env, so assign it // manually for the Slack commit link below. It also has no refspec option, so // SG_COMMIT must name a branch or tag - a bare SHA won't resolve. Fine for this job; - // integration-test/e2e/Jenkinsfile has the checkout+refspec form if that changes. + // .ci/jenkins/e2e/Jenkinsfile has the checkout+refspec form if that changes. def scmVars = git( url: 'git@github.com:couchbase/sync_gateway.git', branch: "${params.SG_COMMIT}", @@ -87,18 +87,24 @@ pipeline { env.GIT_COMMIT = scmVars.GIT_COMMIT } // SG_COMMIT can in principle be overridden to an arbitrary branch, whose copies of these - // helpers may be stale or missing methods the steps below need - always load the current - // ones from main instead. + // helpers and of the test runner may be stale or missing what the steps below need - + // always fetch the current ones from main instead. sh ''' set -eu - curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/jenkinsHelpers.groovy -o integration-test/jenkinsHelpers.groovy + mkdir -p .ci/jenkins + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkinsHelpers.groovy -o .ci/jenkinsHelpers.groovy + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/slack_usernames.yaml -o .ci/slack_usernames.yaml + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkins/integration-build.sh -o .ci/jenkins/integration-build.sh + # integration-build.sh calls this, and SG_COMMIT may predate it, so take it from main too + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh + chmod +x .ci/jenkins/integration-build.sh .ci/retry.sh ''' script { // Posted after the fetch above, and after the checkout since the status needs a SHA. notifyGitHub('running') // The build history lists the build number alone, so name what identifies a run // here, the same way SyncGatewayIntegration/Jenkinsfile does. - def trigger = load('integration-test/jenkinsHelpers.groovy').triggeredBy() + def trigger = load('.ci/jenkinsHelpers.groovy').triggeredBy() currentBuild.description = [params.SG_COMMIT, trigger ? "for ${trigger}" : ''].findAll { it }.join(' ') } } @@ -107,14 +113,14 @@ pipeline { steps { sshagent(credentials: ['CB_SG_Robot_Github_SSH_Key']) { script { - // jenkins-integration-build.sh separates test failures (exit 50 -> UNSTABLE) from a + // .ci/jenkins/integration-build.sh separates test failures (exit 50 -> UNSTABLE) from a // test setup failure or unrecovered panic (exit 1 -> FAILURE). A plain sh step would // collapse both into FAILURE, so the unstable post block below would never run. - def exitCode = sh(script: './jenkins-integration-build.sh -m', returnStatus: true) + def exitCode = sh(script: './.ci/jenkins/integration-build.sh -m', returnStatus: true) if (exitCode == 50) { unstable('At least one integration test failed') } else if (exitCode != 0) { - error("jenkins-integration-build.sh exited ${exitCode}") + error(".ci/jenkins/integration-build.sh exited ${exitCode}") } } } diff --git a/integration-test/SyncGatewayIntegration/Jenkinsfile b/.ci/jenkins/SyncGatewayIntegration/Jenkinsfile similarity index 88% rename from integration-test/SyncGatewayIntegration/Jenkinsfile rename to .ci/jenkins/SyncGatewayIntegration/Jenkinsfile index 117259a308..da9d627740 100644 --- a/integration-test/SyncGatewayIntegration/Jenkinsfile +++ b/.ci/jenkins/SyncGatewayIntegration/Jenkinsfile @@ -3,7 +3,7 @@ // Note for whoever creates this job: its *first* build runs with no parameter values at all. Jenkins // registers the parameters{} block when it first evaluates this file, but only applies the defaults from the -// second build onwards, so that first run dies in jenkins-integration-build.sh's REQUIRED_VARS check with +// second build onwards, so that first run dies in .ci/jenkins/integration-build.sh's REQUIRED_VARS check with // "COUCHBASE_SERVER_PROTOCOL environment variable is required to be set". Just run it again - it isn't a // misconfiguration. @@ -26,11 +26,11 @@ def slackDetails(helpers) { // post block, which would leave the failure unreported entirely. def notifySlack(String status) { def color = ['passed': 'good', 'aborted': 'warning'].get(status, 'danger') - if (!fileExists('integration-test/jenkinsHelpers.groovy')) { + if (!fileExists('.ci/jenkinsHelpers.groovy')) { slackSend(channel: 'syncgatewaybot', color: color, message: ":x: *SyncGatewayIntegration ${status}* - ${currentBuild.fullDisplayName}\n${env.BUILD_URL}") return } - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') if (status == 'aborted' && helpers.isSelfCancelledAdhocBuild()) { echo('Build was started and cancelled by the same user - skipping Slack notification') return @@ -49,11 +49,11 @@ def notifySlack(String status) { // failed before that point - skip rather than throwing out of a post block, which would leave the result // unreported entirely. def notifyGitHub(String status) { - if (!fileExists('integration-test/jenkinsHelpers.groovy')) { - echo('integration-test/jenkinsHelpers.groovy not present - skipping GitHub commit status') + if (!fileExists('.ci/jenkinsHelpers.groovy')) { + echo('.ci/jenkinsHelpers.groovy not present - skipping GitHub commit status') return } - load('integration-test/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-integration-test', 'Integration tests', status) + load('.ci/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-integration-test', 'Integration tests', status) } pipeline { @@ -95,7 +95,7 @@ To use pre-release versions, find the number on https://github.com/orgs/cb-vanil ) string(name: 'SG_TEST_BUCKET_POOL_SIZE', defaultValue: '4', description: 'Number of buckets to create in by the test harness.') string(name: 'RUN_COUNT', defaultValue: '', description: 'Adds -count parameter to test runs. Default is not specified.') - // jenkins-integration-build.sh reads RUN_WALRUS unconditionally under `set -u`, so it has to be set. + // .ci/jenkins/integration-build.sh reads RUN_WALRUS unconditionally under `set -u`, so it has to be set. // Declaring it here is enough: Jenkins exposes parameters as environment variables. booleanParam(name: 'RUN_WALRUS', defaultValue: false, description: 'Run rosmar tests in addition to the Couchbase Server tests.') booleanParam(name: 'SG_TEST_BUCKET_POOL_DEBUG', defaultValue: false, description: 'This will log the test bucket pool creation and cleanup.') @@ -143,12 +143,18 @@ To use pre-release versions, find the number on https://github.com/orgs/cb-vanil env.GIT_COMMIT = scmVars.GIT_COMMIT env.GIT_BRANCH = scmVars.GIT_BRANCH } - // SG_COMMIT can be an arbitrary developer branch, whose copies of these helpers may be - // stale or missing methods the steps below need - always load the current ones from main - // instead. + // SG_COMMIT can be an arbitrary developer branch, whose copies of these helpers and of + // the test runner may be stale or missing what the steps below need - always fetch the + // current ones from main instead. sh ''' set -eu - curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/jenkinsHelpers.groovy -o integration-test/jenkinsHelpers.groovy + mkdir -p .ci/jenkins + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkinsHelpers.groovy -o .ci/jenkinsHelpers.groovy + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/slack_usernames.yaml -o .ci/slack_usernames.yaml + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkins/integration-build.sh -o .ci/jenkins/integration-build.sh + # integration-build.sh calls this, and SG_COMMIT may predate it, so take it from main too + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh + chmod +x .ci/jenkins/integration-build.sh .ci/retry.sh ''' script { // Posted after the fetch above, and after the checkout since the status needs a SHA. @@ -170,7 +176,7 @@ To use pre-release versions, find the number on https://github.com/orgs/cb-vanil sshagent(credentials: ['CB_SG_Robot_Github_SSH_Key']) { withCredentials([usernamePassword(credentialsId: 'github_ghcr_pat', usernameVariable: 'GHCR_USER', passwordVariable: 'GHCR_TOKEN')]) { script { - // `set -e` propagates jenkins-integration-build.sh's exit status, which separates + // `set -e` propagates .ci/jenkins/integration-build.sh's exit status, which separates // test failures (50 -> UNSTABLE) from a test setup failure or unrecovered panic // (1 -> FAILURE). A plain sh step would collapse both into FAILURE, so the // unstable post block below would never run. @@ -183,7 +189,7 @@ To use pre-release versions, find the number on https://github.com/orgs/cb-vanil IMAGE="couchbase/server:${COUCHBASE_SERVER_VERSION}" fi docker pull "$IMAGE" - ./jenkins-integration-build.sh + ./.ci/jenkins/integration-build.sh ''') if (exitCode == 50) { unstable('At least one integration test failed') diff --git a/integration-test/SyncGatewayIntegrationWeekly/Jenkinsfile b/.ci/jenkins/SyncGatewayIntegrationWeekly/Jenkinsfile similarity index 100% rename from integration-test/SyncGatewayIntegrationWeekly/Jenkinsfile rename to .ci/jenkins/SyncGatewayIntegrationWeekly/Jenkinsfile diff --git a/integration-test/TopologyTests/Jenkinsfile b/.ci/jenkins/TopologyTests/Jenkinsfile similarity index 80% rename from integration-test/TopologyTests/Jenkinsfile rename to .ci/jenkins/TopologyTests/Jenkinsfile index e3731b72b9..c9d817830a 100644 --- a/integration-test/TopologyTests/Jenkinsfile +++ b/.ci/jenkins/TopologyTests/Jenkinsfile @@ -4,7 +4,7 @@ // Note for whoever creates this job: its *first* build runs with no parameter values at all. Jenkins // registers the parameters{} block when it first evaluates this file, but only applies the defaults from the -// second build onwards, so that first run dies in jenkins-integration-build.sh's REQUIRED_VARS check with +// second build onwards, so that first run dies in .ci/jenkins/integration-build.sh's REQUIRED_VARS check with // "COUCHBASE_SERVER_PROTOCOL environment variable is required to be set". Just run it again - it isn't a // misconfiguration. @@ -27,11 +27,11 @@ def slackDetails(helpers) { // post block, which would leave the failure unreported entirely. def notifySlack(String status) { def color = ['passed': 'good', 'aborted': 'warning'].get(status, 'danger') - if (!fileExists('integration-test/jenkinsHelpers.groovy')) { + if (!fileExists('.ci/jenkinsHelpers.groovy')) { slackSend(channel: 'syncgatewaybot', color: color, message: ":x: *TopologyTests ${status}* - ${currentBuild.fullDisplayName}\n${env.BUILD_URL}") return } - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') if (status == 'aborted' && helpers.isSelfCancelledAdhocBuild()) { echo('Build was started and cancelled by the same user - skipping Slack notification') return @@ -50,11 +50,11 @@ def notifySlack(String status) { // failed before that point - skip rather than throwing out of a post block, which would leave the result // unreported entirely. def notifyGitHub(String status) { - if (!fileExists('integration-test/jenkinsHelpers.groovy')) { - echo('integration-test/jenkinsHelpers.groovy not present - skipping GitHub commit status') + if (!fileExists('.ci/jenkinsHelpers.groovy')) { + echo('.ci/jenkinsHelpers.groovy not present - skipping GitHub commit status') return } - load('integration-test/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-topology-tests', 'Topology tests', status) + load('.ci/jenkinsHelpers.groovy').notifyCommitStatus('sgw-jenkins-topology-tests', 'Topology tests', status) } pipeline { @@ -87,7 +87,7 @@ pipeline { } environment { - // jenkins-integration-build.sh reads RUN_WALRUS unconditionally under `set -u`, so it has to be + // .ci/jenkins/integration-build.sh reads RUN_WALRUS unconditionally under `set -u`, so it has to be // set even though this job never runs the Rosmar leg. RUN_WALRUS = 'false' TARGET_PACKAGE = 'topologytest' @@ -102,7 +102,7 @@ pipeline { // The `git` step only returns GIT_COMMIT rather than populating env, so assign it // manually for the Slack commit link below. It also has no refspec option, so // SG_COMMIT must name a branch or tag - a bare SHA won't resolve. Fine for this job; - // integration-test/e2e/Jenkinsfile has the checkout+refspec form if that changes. + // .ci/jenkins/e2e/Jenkinsfile has the checkout+refspec form if that changes. def scmVars = git( url: 'git@github.com:couchbase/sync_gateway.git', branch: "${params.SG_COMMIT}", @@ -110,19 +110,25 @@ pipeline { ) env.GIT_COMMIT = scmVars.GIT_COMMIT } - // SG_COMMIT can be an arbitrary developer branch, whose copies of these helpers may be - // stale or missing methods the steps below need - always load the current ones from main - // instead. + // SG_COMMIT can be an arbitrary developer branch, whose copies of these helpers and of + // the test runner may be stale or missing what the steps below need - always fetch the + // current ones from main instead. sh ''' set -eu - curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/jenkinsHelpers.groovy -o integration-test/jenkinsHelpers.groovy + mkdir -p .ci/jenkins + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkinsHelpers.groovy -o .ci/jenkinsHelpers.groovy + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/slack_usernames.yaml -o .ci/slack_usernames.yaml + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkins/integration-build.sh -o .ci/jenkins/integration-build.sh + # integration-build.sh calls this, and SG_COMMIT may predate it, so take it from main too + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh + chmod +x .ci/jenkins/integration-build.sh .ci/retry.sh ''' script { // Posted after the fetch above, and after the checkout since the status needs a SHA. notifyGitHub('running') // The build history lists the build number alone, so name the parameters that // identify a run here, the same way SyncGatewayIntegration/Jenkinsfile does. - def trigger = load('integration-test/jenkinsHelpers.groovy').triggeredBy() + def trigger = load('.ci/jenkinsHelpers.groovy').triggeredBy() def targetTest = params.TARGET_TEST == 'ALL' ? '' : params.TARGET_TEST currentBuild.description = [ params.SG_COMMIT, @@ -137,14 +143,14 @@ pipeline { steps { sshagent(credentials: ['CB_SG_Robot_Github_SSH_Key']) { script { - // jenkins-integration-build.sh separates test failures (exit 50 -> UNSTABLE) from a + // .ci/jenkins/integration-build.sh separates test failures (exit 50 -> UNSTABLE) from a // test setup failure or unrecovered panic (exit 1 -> FAILURE). A plain sh step would // collapse both into FAILURE, so the unstable post block below would never run. - def exitCode = sh(script: './jenkins-integration-build.sh', returnStatus: true) + def exitCode = sh(script: './.ci/jenkins/integration-build.sh', returnStatus: true) if (exitCode == 50) { unstable('At least one topology test failed') } else if (exitCode != 0) { - error("jenkins-integration-build.sh exited ${exitCode}") + error(".ci/jenkins/integration-build.sh exited ${exitCode}") } } } diff --git a/integration-test/e2e/Jenkinsfile b/.ci/jenkins/e2e/Jenkinsfile similarity index 92% rename from integration-test/e2e/Jenkinsfile rename to .ci/jenkins/e2e/Jenkinsfile index 245d4be611..a4cc24490d 100644 --- a/integration-test/e2e/Jenkinsfile +++ b/.ci/jenkins/e2e/Jenkinsfile @@ -1,5 +1,5 @@ // Details block included in every Slack message for this pipeline (see jenkinsHelpers.groovy). -// The Checkout stage overwrites integration-test/jenkinsHelpers.groovy and .github/slack_usernames.yaml with +// The Checkout stage overwrites .ci/jenkinsHelpers.groovy and .ci/slack_usernames.yaml with // the latest versions from main, so both stay current regardless of which SG_COMMIT is under test. def e2eDetails(helpers) { return [ @@ -17,11 +17,11 @@ def e2eDetails(helpers) { // build failed before that point - skip rather than throwing out of a post block, which would leave the // result unreported entirely. def notifyGitHub(String status) { - if (!fileExists('integration-test/jenkinsHelpers.groovy')) { - echo('integration-test/jenkinsHelpers.groovy not present - skipping GitHub commit status') + if (!fileExists('.ci/jenkinsHelpers.groovy')) { + echo('.ci/jenkinsHelpers.groovy not present - skipping GitHub commit status') return } - load('integration-test/jenkinsHelpers.groovy').notifyCommitStatus("sgw-jenkins-e2e-test-${params.BACKING_STORE}", 'E2E tests', status) + load('.ci/jenkinsHelpers.groovy').notifyCommitStatus("sgw-jenkins-e2e-test-${params.BACKING_STORE}", 'E2E tests', status) } pipeline { @@ -152,20 +152,20 @@ pipeline { // ones from main instead. sh ''' set -eu - mkdir -p integration-test/e2e .github .ci + mkdir -p integration-test/e2e .ci curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/e2e/run_e2e_tests.sh -o integration-test/e2e/run_e2e_tests.sh # run_e2e_tests.sh calls this, and SG_COMMIT may predate it, so take it from main too curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/retry.sh -o .ci/retry.sh chmod +x .ci/retry.sh - curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/integration-test/jenkinsHelpers.groovy -o integration-test/jenkinsHelpers.groovy - curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.github/slack_usernames.yaml -o .github/slack_usernames.yaml + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/jenkinsHelpers.groovy -o .ci/jenkinsHelpers.groovy + curl -sSf https://raw.githubusercontent.com/couchbase/sync_gateway/main/.ci/slack_usernames.yaml -o .ci/slack_usernames.yaml ''' script { // Posted after the fetch above, and after the checkout since the status needs a SHA. notifyGitHub('running') // The build history lists the build number alone, so name the parameters that // identify a run here, the same way SyncGatewayIntegration/Jenkinsfile does. - def trigger = load('integration-test/jenkinsHelpers.groovy').triggeredBy() + def trigger = load('.ci/jenkinsHelpers.groovy').triggeredBy() def backingStore = params.BACKING_STORE == 'cbs' ? "cbs ${params.COUCHBASE_SERVER_VERSION}" : params.BACKING_STORE currentBuild.description = [ params.SG_COMMIT, @@ -206,7 +206,7 @@ pipeline { success { script { notifyGitHub('passed') - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') def message = helpers.slackMessage('E2E tests', 'passed', env.BUILD_URL, e2eDetails(helpers)) slackSend(channel: 'syncgatewaybot', color: 'good', message: message) def dmTarget = helpers.slackUserIdForBuild() @@ -218,14 +218,14 @@ pipeline { unstable { script { notifyGitHub('unstable') - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') helpers.slackSendFailure('E2E tests', 'unstable', env.BUILD_URL, e2eDetails(helpers)) } } failure { script { notifyGitHub('build failure') - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') helpers.slackSendFailure('E2E tests', 'failed', "Console: ${env.BUILD_URL}console", e2eDetails(helpers)) } } diff --git a/jenkins-integration-build.sh b/.ci/jenkins/integration-build.sh similarity index 100% rename from jenkins-integration-build.sh rename to .ci/jenkins/integration-build.sh diff --git a/jenkins/update-git-docs-tag/Jenkinsfile b/.ci/jenkins/update-git-docs-tag/Jenkinsfile similarity index 100% rename from jenkins/update-git-docs-tag/Jenkinsfile rename to .ci/jenkins/update-git-docs-tag/Jenkinsfile diff --git a/integration-test/jenkinsHelpers.groovy b/.ci/jenkinsHelpers.groovy similarity index 94% rename from integration-test/jenkinsHelpers.groovy rename to .ci/jenkinsHelpers.groovy index d9e3166aec..937f421946 100644 --- a/integration-test/jenkinsHelpers.groovy +++ b/.ci/jenkinsHelpers.groovy @@ -8,7 +8,7 @@ // Shared Jenkins pipeline helpers - Slack notifications, GitHub commit statuses and build metadata - // loaded via the `load` step by the Jenkinsfiles in this repo (see Jenkinsfile and -// integration-test/*/Jenkinsfile). +// .ci/jenkins/*/Jenkinsfile). // Builds a Slack-friendly summary of the JUnit results recorded by the 'junit' step in post.always, // including up to 10 failed test names (Jenkins runs post.always before success/failure/unstable/aborted, @@ -101,7 +101,7 @@ def triggeredBy() { } // Looks up the Slack member ID of whoever manually triggered this build in the UI, via -// .github/slack_usernames.yaml (Jenkins usernames are identical to GitHub usernames in this org). +// .ci/slack_usernames.yaml (Jenkins usernames are identical to GitHub usernames in this org). // Returns null for non-user-triggered builds (e.g. an automatic fan-out from an upstream job, // which has an UpstreamCause instead) or if the triggering user has no entry/Slack ID in the map. def slackUserIdForBuild() { @@ -110,15 +110,15 @@ def slackUserIdForBuild() { echo('No UserIdCause on this build (not manually triggered) - skipping Slack DM') return null } - if (!fileExists('.github/slack_usernames.yaml')) { - echo('.github/slack_usernames.yaml not present (build likely failed before it could be fetched) - skipping Slack DM') + if (!fileExists('.ci/slack_usernames.yaml')) { + echo('.ci/slack_usernames.yaml not present (build likely failed before it could be fetched) - skipping Slack DM') return null } def githubUsername = userIdCauses[0].userId - def slackMap = readYaml(file: '.github/slack_usernames.yaml') + def slackMap = readYaml(file: '.ci/slack_usernames.yaml') def slackUserId = slackMap[githubUsername] if (!slackUserId) { - echo("No Slack ID mapped for '${githubUsername}' in .github/slack_usernames.yaml") + echo("No Slack ID mapped for '${githubUsername}' in .ci/slack_usernames.yaml") return null } return slackUserId diff --git a/.github/slack_usernames.yaml b/.ci/slack_usernames.yaml similarity index 100% rename from .github/slack_usernames.yaml rename to .ci/slack_usernames.yaml diff --git a/.github/workflows/shell.yml b/.github/workflows/shell.yml index 20442aca04..216c875955 100644 --- a/.github/workflows/shell.yml +++ b/.github/workflows/shell.yml @@ -12,10 +12,11 @@ permissions: on: push: - # Only run when we change a shell script + # Only run when we change a shell script. '*' does not cross '/' in a path + # filter, so '**.sh' is what reaches scripts outside the repo root. paths: - - "*.sh" - - ".ci/*.sh" + - "**.sh" + - ".github/workflows/shell.yml" branches: - "main" - "release/*" @@ -23,10 +24,11 @@ on: - "CBG*" - "ci-*" pull_request: - # Only run when we change a shell script + # Only run when we change a shell script. '*' does not cross '/' in a path + # filter, so '**.sh' is what reaches scripts outside the repo root. paths: - - "*.sh" - - ".ci/*.sh" + - "**.sh" + - ".github/workflows/shell.yml" branches: - "main" - "release/*" diff --git a/AGENTS.md b/AGENTS.md index c688b15ce9..a61a971107 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,10 +12,11 @@ go test -tags cb_sg_enterprise,cb_sg_devmode ./... # unit tes - **Enterprise Edition is the default** — pass the `cb_sg_enterprise,cb_sg_devmode` build tags on every Go command unless you were specifically asked to test Community Edition. EE also needs SSH access to a private repo — see [docs/BUILD.md](docs/BUILD.md). - **Community Edition** is what you get with no build tags: `go build -o bin/sync_gateway .` / `go test ./...`. Only drop the tags when CE behaviour is what's being tested. - **If the EE dependencies can't be fetched** (no SSH access to the private repo), fall back to CE and say explicitly that the results are CE-only. -- **CI runs both**: GitHub Actions (`.github/workflows/ci.yml`) builds and tests CE; Jenkins (`Jenkinsfile`, `jenkins-integration-build.sh`) runs EE. +- **CI runs both**: GitHub Actions (`.github/workflows/ci.yml`) builds and tests CE; Jenkins (`Jenkinsfile`, `.ci/jenkins/integration-build.sh`) runs EE. - **Integration tests** against a real Couchbase Server are covered in [docs/TESTING.md](docs/TESTING.md): starting a local cluster with `integration-test/start_cbs.py`, the `SG_TEST_*` environment variables, and the bucket pool. - **Python tooling** (`tools/`): See [tools/AGENTS.md](tools/AGENTS.md). - **Lint**: CI enforces `.golangci-strict.yml`; reproduce it locally with `pre-commit run golangci-lint --all-files`. Some conventions are enforced here rather than written down — the linter message explains the fix. +- **CI layout**: everything CI-only lives in `.ci/` — `retry.sh`, the shared pipeline helpers (`jenkinsHelpers.groovy`, `slack_usernames.yaml`), and the Jenkins job definitions under `.ci/jenkins/`. Scripts meant to also be run by hand (`build.sh`, `test.sh`, `integration-test/*.sh`, `integration-test/start_cbs.py`) stay where they are. The root `Jenkinsfile` stays put because Jenkins multibranch pipelines discover it there. - **CI tools** are pinned by `tool` directives in `go.mod` and run as `go tool ` — `gotestsum`, `addlicense`, `goimports`, `goveralls`. No install step is needed. `cbdinocluster` is pinned in its own module: `go -C integration-test/tools tool cbdinocluster`. Git: `main` is the current in-development version. Released versions and backports live in `release/x.y.z` branches. Feature branches are named `CBG-xxxx` after the Jira ticket. diff --git a/Jenkinsfile b/Jenkinsfile index 6493ffd7e0..5ddbae7d51 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -246,7 +246,7 @@ pipeline { // archive non-verbose outputs upon failure for inspection (each verbose output is conditionally archived on stage failure) archiveArtifacts excludes: 'verbose_*.out', artifacts: '*.out', fingerprint: false, allowEmptyArchive: true script { - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') helpers.slackSendFailure('main SGW pipeline', 'unstable', env.BUILD_URL, ['Commit': helpers.gitCommitLink()]) } } @@ -254,14 +254,14 @@ pipeline { // archive non-verbose outputs upon failure for inspection (each verbose output is conditionally archived on stage failure) archiveArtifacts excludes: 'verbose_*.out', artifacts: '*.out', fingerprint: false, allowEmptyArchive: true script { - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') helpers.slackSendFailure('main SGW pipeline', 'build failure', env.BUILD_URL, ['Commit': helpers.gitCommitLink()]) } } aborted { archiveArtifacts excludes: 'verbose_*.out', artifacts: '*.out', fingerprint: false, allowEmptyArchive: true script { - def helpers = load('integration-test/jenkinsHelpers.groovy') + def helpers = load('.ci/jenkinsHelpers.groovy') helpers.slackSendFailure('main SGW pipeline', 'unstable', env.BUILD_URL, ['Commit': helpers.gitCommitLink()]) } }