From f41001ac410dd476223ba462619aea80952a4b92 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Oct 2025 12:34:49 +0100 Subject: [PATCH 1/4] feat(ci): support exhaustive tests for PRs (code changes or GH comment) (#18327) --- .buildkite/pull-requests.json | 90 +++++++++++++------ .../smart_exhaustive_tests_pipeline.yml | 34 +++++++ catalog-info.yaml | 51 +++++++++++ 3 files changed, 150 insertions(+), 25 deletions(-) create mode 100644 .buildkite/smart_exhaustive_tests_pipeline.yml diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 2414a75b5b..a01d003499 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -1,26 +1,66 @@ { - "jobs": [ - { - "enabled": true, - "pipeline_slug": "logstash-pull-request-pipeline", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]", "elastic-vault-github-plugin-prod[bot]"], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", - "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ - "^.github/", - "^docs/", - "^.mergify.yml$", - "^.pre-commit-config.yaml", - "\\.md$" - ], - "always_require_ci_on_changed": [ ] - } - ] - } + "jobs": [ + { + "enabled": true, + "pipeline_slug": "logstash-pull-request-pipeline", + "allow_org_users": true, + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [ + "dependabot[bot]", + "mergify[bot]", + "github-actions[bot]", + "elastic-vault-github-plugin-prod[bot]" + ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", + "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [ + "^.github/", + "^docs/", + "^.mergify.yml$", + "^.pre-commit-config.yaml", + "\\.md$" + ], + "always_require_ci_on_changed": [] + }, + { + "enabled": true, + "pipeline_slug": "logstash-smart-exhaustive-tests-pipeline", + "allow_org_users": true, + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [ + "dependabot[bot]", + "mergify[bot]", + "github-actions[bot]" + ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/?run\s+exhaustive\s+tests$", + "always_trigger_comment_regex": "^/?run\s+exhaustive\s+tests$", + "always_trigger_comment_regex": "^(?:(?:/run\\W+)(?:exhaustive)\\W+(?:tests))", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [ + "^.github/", + "^docs/", + "^.mergify.yml$", + "^.pre-commit-config.yaml", + "\\.md$" + ], + "always_require_ci_on_changed": [ + "^qa/acceptance/" + ] + } + ] +} \ No newline at end of file diff --git a/.buildkite/smart_exhaustive_tests_pipeline.yml b/.buildkite/smart_exhaustive_tests_pipeline.yml new file mode 100644 index 0000000000..3b8252c3ad --- /dev/null +++ b/.buildkite/smart_exhaustive_tests_pipeline.yml @@ -0,0 +1,34 @@ +steps: + - label: "Trigger logstash-exhaustive-tests-pipeline for PRs with qa/acceptance/ changes" + if: build.pull_request.id != null + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + interpolation: false + watch: + - path: + - ^qa/acceptance/ + - ^ci/acceptance_tests.sh + - .buildkite/smart_exhaustive_tests_pipeline.yml + config: + trigger: "logstash-exhaustive-tests-pipeline" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - ELASTIC_SLACK_NOTIFICATIONS_ENABLE=false + + - label: "Trigger logstash-exhaustive-tests-pipeline for GitHub comments" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") != "" + trigger: "logstash-exhaustive-tests-pipeline" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - ELASTIC_SLACK_NOTIFICATIONS_ENABLE=false diff --git a/catalog-info.yaml b/catalog-info.yaml index 6d7abe61aa..27350e8ef7 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -478,6 +478,57 @@ spec: # SECTION END: Exhaustive tests pipeline # ************************************** +# **************************************** +# SECTION START: Smart exhaustive tests pipeline +# **************************************** + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-smart-exhaustive-tests-pipeline + description: 'Logstash Smart Exhaustive tests pipeline' + links: + - title: 'Logstash Smart Exhaustive tests pipeline' + url: https://buildkite.com/elastic/logstash-smart-exhaustive-tests-pipeline +spec: + type: buildkite-pipeline + owner: group:logstash + system: platform-ingest + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: "Logstash Smart Exhaustive tests pipeline" + description: '🔍 Run smart exhaustive tests against Logstash using different operating systems' + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/smart_exhaustive_tests_pipeline.yml" + provider_settings: + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_branches: false + build_tags: false + filter_enabled: true + filter_condition: >- + build.creator.name == 'elasticmachine' && build.pull_request.id != null + cancel_intermediate_builds: true + skip_intermediate_builds: true + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +# ************************************** +# SECTION END: Smart exhaustive tests pipeline +# ************************************** + # ******************************************** # Declare supported plugin tests pipeline # ******************************************** From 410d62c529821c1584f8318e0ba30de4b664f489 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 30 Oct 2025 16:44:56 +0100 Subject: [PATCH 2/4] bk: fix regex and add new file (#18382) --- .buildkite/pull-requests.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index a01d003499..25b4cd93b1 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -46,9 +46,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/?run\s+exhaustive\s+tests$", - "always_trigger_comment_regex": "^/?run\s+exhaustive\s+tests$", - "always_trigger_comment_regex": "^(?:(?:/run\\W+)(?:exhaustive)\\W+(?:tests))", + "trigger_comment_regex": "^/?run\\s+exhaustive\\s+tests$", + "always_trigger_comment_regex": "^/?run\\s+exhaustive\\s+tests$", "skip_ci_labels": [], "skip_target_branches": [], "skip_ci_on_only_changed": [ @@ -59,7 +58,8 @@ "\\.md$" ], "always_require_ci_on_changed": [ - "^qa/acceptance/" + "^qa/acceptance/", + "^ci/acceptance_tests.sh" ] } ] From 426fce49b9d63cf3e4a2f638741b823e078ae9b8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Oct 2025 10:07:54 +0100 Subject: [PATCH 3/4] bk(smart exhaustive tests): tune the steps to support GH comments and changesets (#18383) --- .../smart_exhaustive_tests_pipeline.yml | 21 ++++++++----------- qa/acceptance/empty.txt | 0 2 files changed, 9 insertions(+), 12 deletions(-) create mode 100644 qa/acceptance/empty.txt diff --git a/.buildkite/smart_exhaustive_tests_pipeline.yml b/.buildkite/smart_exhaustive_tests_pipeline.yml index 3b8252c3ad..d19c6c7442 100644 --- a/.buildkite/smart_exhaustive_tests_pipeline.yml +++ b/.buildkite/smart_exhaustive_tests_pipeline.yml @@ -1,6 +1,6 @@ steps: - label: "Trigger logstash-exhaustive-tests-pipeline for PRs with qa/acceptance/ changes" - if: build.pull_request.id != null + if: build.pull_request.id != null && build.env("GITHUB_PR_TRIGGER_COMMENT") != "/run exhaustive tests" plugins: - monorepo-diff#v1.0.1: diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" @@ -16,19 +16,16 @@ steps: commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" env: - - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} - - ELASTIC_SLACK_NOTIFICATIONS_ENABLE=false + BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}" + BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: false - label: "Trigger logstash-exhaustive-tests-pipeline for GitHub comments" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") != "" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "/run exhaustive tests" trigger: "logstash-exhaustive-tests-pipeline" build: - commit: "${BUILDKITE_COMMIT}" - branch: "${BUILDKITE_BRANCH}" + commit: "HEAD" + branch: "pull/${BUILDKITE_PULL_REQUEST}/merge" env: - - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} - - ELASTIC_SLACK_NOTIFICATIONS_ENABLE=false + BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}" + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: false diff --git a/qa/acceptance/empty.txt b/qa/acceptance/empty.txt new file mode 100644 index 0000000000..e69de29bb2 From 1555bd8a7dbfa227cdde3474f42736ea271abd3f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Oct 2025 11:48:28 +0100 Subject: [PATCH 4/4] chore: remove test file for exhaustive tests and normalise github commands (#18384) --- .buildkite/pull-requests.json | 4 ++-- .buildkite/smart_exhaustive_tests_pipeline.yml | 4 ++-- qa/acceptance/empty.txt | 0 3 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 qa/acceptance/empty.txt diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 25b4cd93b1..30e77d561a 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -46,8 +46,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/?run\\s+exhaustive\\s+tests$", - "always_trigger_comment_regex": "^/?run\\s+exhaustive\\s+tests$", + "trigger_comment_regex": "^run\\s+exhaustive\\s+tests$", + "always_trigger_comment_regex": "^run\\s+exhaustive\\s+tests$", "skip_ci_labels": [], "skip_target_branches": [], "skip_ci_on_only_changed": [ diff --git a/.buildkite/smart_exhaustive_tests_pipeline.yml b/.buildkite/smart_exhaustive_tests_pipeline.yml index d19c6c7442..61021283f4 100644 --- a/.buildkite/smart_exhaustive_tests_pipeline.yml +++ b/.buildkite/smart_exhaustive_tests_pipeline.yml @@ -1,6 +1,6 @@ steps: - label: "Trigger logstash-exhaustive-tests-pipeline for PRs with qa/acceptance/ changes" - if: build.pull_request.id != null && build.env("GITHUB_PR_TRIGGER_COMMENT") != "/run exhaustive tests" + if: build.pull_request.id != null && build.env("GITHUB_PR_TRIGGER_COMMENT") != "run exhaustive tests" plugins: - monorepo-diff#v1.0.1: diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" @@ -21,7 +21,7 @@ steps: ELASTIC_SLACK_NOTIFICATIONS_ENABLED: false - label: "Trigger logstash-exhaustive-tests-pipeline for GitHub comments" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "/run exhaustive tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "run exhaustive tests" trigger: "logstash-exhaustive-tests-pipeline" build: commit: "HEAD" diff --git a/qa/acceptance/empty.txt b/qa/acceptance/empty.txt deleted file mode 100644 index e69de29bb2..0000000000