diff --git a/.buildkite/aarch64_pipeline.yml b/.buildkite/aarch64_pipeline.yml index 3ab0e7ce1e..e99b838375 100644 --- a/.buildkite/aarch64_pipeline.yml +++ b/.buildkite/aarch64_pipeline.yml @@ -23,9 +23,6 @@ steps: - label: ":java: Java unit tests" key: "java-unit-tests" - env: - # https://github.com/elastic/logstash/pull/15486 for background - ENABLE_SONARQUBE: "false" command: | set -euo pipefail diff --git a/.buildkite/jdk_availability_check_pipeline.yml b/.buildkite/jdk_availability_check_pipeline.yml deleted file mode 100644 index 3fa826564e..0000000000 --- a/.buildkite/jdk_availability_check_pipeline.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: - - label: "JDK Availability check" - key: "jdk-availability-check" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci" - cpu: "4" - memory: "6Gi" - ephemeralStorage: "100Gi" - command: | - set -euo pipefail - - source .buildkite/scripts/common/container-agent.sh - export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info" - ci/check_jdk_version_availability.sh \ No newline at end of file diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 2414a75b5b..bd9a7d2200 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -1,26 +1,65 @@ { - "jobs": [ - { - "enabled": true, - "pipeline_slug": "logstash-pull-request-pipeline", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]", "elastic-vault-github-plugin-prod[bot]"], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", - "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ - "^.github/", - "^docs/", - "^.mergify.yml$", - "^.pre-commit-config.yaml", - "\\.md$" - ], - "always_require_ci_on_changed": [ ] - } - ] - } + "jobs": [ + { + "enabled": true, + "pipeline_slug": "logstash-pull-request-pipeline", + "allow_org_users": true, + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [ + "dependabot[bot]", + "mergify[bot]", + "github-actions[bot]", + "elastic-vault-github-plugin-prod[bot]" + ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", + "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [ + "^.github/", + "^docs/", + "^.mergify.yml$", + "^.pre-commit-config.yaml", + "\\.md$" + ], + "always_require_ci_on_changed": [] + }, + { + "enabled": true, + "pipeline_slug": "logstash-smart-exhaustive-tests-pipeline", + "allow_org_users": true, + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [ + "dependabot[bot]", + "mergify[bot]", + "github-actions[bot]" + ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^(?:(?:/run\\W+)?(?:exhaustive)\\W+(?:tests))", + "always_trigger_comment_regex": "^(?:(?:/run\\W+)(?:exhaustive)\\W+(?:tests))", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [ + "^.github/", + "^docs/", + "^.mergify.yml$", + "^.pre-commit-config.yaml", + "\\.md$" + ], + "always_require_ci_on_changed": [ + "^qa/acceptance/" + ] + } + ] +} diff --git a/.buildkite/pull_request_pipeline.yml b/.buildkite/pull_request_pipeline.yml index 19e1a9f925..b5f62f995e 100644 --- a/.buildkite/pull_request_pipeline.yml +++ b/.buildkite/pull_request_pipeline.yml @@ -88,8 +88,6 @@ steps: retry: automatic: - limit: 3 - env: - ENABLE_SONARQUBE: true command: | set -euo pipefail source .buildkite/scripts/common/container-agent.sh @@ -110,8 +108,6 @@ steps: retry: automatic: - limit: 3 - env: - ENABLE_SONARQUBE: true command: | set -euo pipefail @@ -122,27 +118,6 @@ steps: - "**/jacocoTestReport.xml" - "**/build/classes/**/*.*" - - label: ":sonarqube: Continuous Code Inspection" - if: | - build.pull_request.id != null || - build.branch == "main" || - build.branch =~ /^[0-9]+\.[0-9]+\$/ - env: - VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/logstash/sonar-analyze-token" - agents: - image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" - command: - - "buildkite-agent artifact download --step ruby-unit-tests coverage/coverage.json ." - - "buildkite-agent artifact download --step java-unit-tests **/jacocoTestReport.xml ." - - "buildkite-agent artifact download --step java-unit-tests **/build/classes/**/*.* ." - - "/scan-source-code.sh" - depends_on: - - "ruby-unit-tests" - - "java-unit-tests" - retry: - manual: - allowed: true - - label: "Observability SRE container smoke test" key: "observability-sre-container-smoke-test" agents: diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh index 0a5ee1998e..3799b0a825 100755 --- a/.buildkite/scripts/dra/build_docker.sh +++ b/.buildkite/scripts/dra/build_docker.sh @@ -24,6 +24,9 @@ esac rake artifact:docker || error "artifact:docker build failed." rake artifact:docker_oss || error "artifact:docker_oss build failed." rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." + +# Generating public dockerfiles is the primary use case for NOT using local artifacts +export LOCAL_ARTIFACTS=false rake artifact:dockerfiles || error "artifact:dockerfiles build failed." STACK_VERSION="$(./$(dirname "$0")/../common/qualified-version.sh)" diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py index cbb4900354..f8e50149a3 100644 --- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py +++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py @@ -229,7 +229,6 @@ def java_unit_test(self) -> JobRetValues: step_name_human = "Java Unit Test" step_key = f"{self.group_key}-java-unit-test" test_command = ''' -export ENABLE_SONARQUBE="false" ci/unit_tests.sh java ''' diff --git a/.buildkite/smart_exhaustive_tests_pipeline.yml b/.buildkite/smart_exhaustive_tests_pipeline.yml new file mode 100644 index 0000000000..0e541f902d --- /dev/null +++ b/.buildkite/smart_exhaustive_tests_pipeline.yml @@ -0,0 +1,33 @@ +steps: + - label: "Trigger logstash-exhaustive-tests-pipeline for PRs with qa/acceptance/ changes" + if: build.pull_request.id != null + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + interpolation: false + watch: + - path: + - ^qa/acceptance/ + - .buildkite/smart_exhaustive_tests_pipeline.yml + config: + trigger: "logstash-exhaustive-tests-pipeline" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - ELASTIC_SLACK_NOTIFICATIONS_ENABLE=false + + - label: "Trigger logstash-exhaustive-tests-pipeline for GitHub comments" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") != "" + trigger: "logstash-exhaustive-tests-pipeline" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - ELASTIC_SLACK_NOTIFICATIONS_ENABLE=false diff --git a/.ci/updatecli/bump-java-version.yml b/.ci/updatecli/bump-java-version.yml new file mode 100644 index 0000000000..62f48760d3 --- /dev/null +++ b/.ci/updatecli/bump-java-version.yml @@ -0,0 +1,56 @@ +--- +name: Update java version file +pipelineid: "logstash/jdk-version-updates-{{ requiredEnv "LOGSTASH_BRANCH" }}" + +scms: + default: + kind: github + spec: + user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + branch: '{{ requiredEnv "LOGSTASH_BRANCH" }}' + commitusingapi: true + force: false + +sources: + jdk_major: + kind: yaml + spec: + file: "versions.yml" + key: "$.bundled_jdk.revision" + transformers: + - findsubmatch: + pattern: '^(\d+)\.\d+\.\d+$' + captureindex: 1 + + latest_jdk_version: + kind: json + spec: + file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_{{ source "jdk_major" }}_linux' + key: 'version' + + latest_jdk_build: + kind: json + spec: + file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_{{ source "jdk_major" }}_linux' + key: 'revision' + +targets: + update_jdk_revision: + name: "Update JDK revision" + kind: yaml + sourceid: latest_jdk_version + spec: + file: versions.yml + key: $.bundled_jdk.revision + + update_jdk_build: + name: "Update JDK build" + kind: yaml + sourceid: latest_jdk_build + spec: + file: versions.yml + key: $.bundled_jdk.build \ No newline at end of file diff --git a/.ci/updatecli/bump-logstash-version.yml b/.ci/updatecli/bump-logstash-version.yml new file mode 100644 index 0000000000..a100a19aeb --- /dev/null +++ b/.ci/updatecli/bump-logstash-version.yml @@ -0,0 +1,81 @@ +--- +name: Update logstash version files +pipelineid: "logstash/version-updates-{{ requiredEnv "LOGSTASH_BRANCH" }}" + +scms: + default: + kind: github + spec: + user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + branch: '{{ requiredEnv "LOGSTASH_BRANCH" }}' + commitusingapi: true + force: false + +actions: + default: + title: 'Bump logstash version {{ requiredEnv "LOGSTASH_VERSION" }}' + kind: github/pullrequest + scmid: default + spec: + automerge: false + labels: + - automation + description: |- + ### What + Update logstash version + +sources: + lock_file_exists: + kind: shell + scmid: default + spec: + command: test -f Gemfile.jruby-3.1.lock.release + +targets: + update_logstash_version: + name: Update logstash version in versions.yml + kind: yaml + disablesourceinput: true + scmid: default + spec: + file: versions.yml + key: $.logstash + value: '{{ requiredEnv "LOGSTASH_VERSION" }}' + + update_logstash_core_version: + name: Update logstash-core version in versions.yml + kind: yaml + disablesourceinput: true + scmid: default + spec: + file: versions.yml + key: $.logstash-core + value: '{{ requiredEnv "LOGSTASH_VERSION" }}' + + update_gemfile_lock_dependency: + name: Update logstash-core dependency in lockfile + kind: file + disablesourceinput: true + scmid: default + dependson: + - 'source#lock_file_exists' + spec: + file: Gemfile.jruby-3.1.lock.release + matchpattern: 'logstash-core \(= [0-9]+\.[0-9]+\.[0-9]+' + replacepattern: 'logstash-core (= {{ requiredEnv "LOGSTASH_VERSION" }}' + + update_gemfile_lock_spec: + name: Update logstash-core spec in lockfile + kind: file + disablesourceinput: true + scmid: default + dependson: + - 'source#lock_file_exists' + spec: + file: Gemfile.jruby-3.1.lock.release + matchpattern: 'logstash-core \([0-9]+\.[0-9]+\.[0-9]+-java\)' + replacepattern: 'logstash-core ({{ requiredEnv "LOGSTASH_VERSION" }}-java)' \ No newline at end of file diff --git a/.ci/updatecli/values.d/scm.yml b/.ci/updatecli/values.d/scm.yml new file mode 100644 index 0000000000..62b6e5c249 --- /dev/null +++ b/.ci/updatecli/values.d/scm.yml @@ -0,0 +1,3 @@ +scm: + owner: elastic + repository: logstash diff --git a/.github/workflows/bump-java-version.yml b/.github/workflows/bump-java-version.yml index ac66267e44..818faf236f 100644 --- a/.github/workflows/bump-java-version.yml +++ b/.github/workflows/bump-java-version.yml @@ -1,19 +1,25 @@ -name: Stub GH action for devoping new workflows [STUB] +name: bump-java-version + on: + schedule: + # Run weekly on Mondays at midnight UTC + - cron: '0 0 * * 1' workflow_dispatch: - pull_request: - types: [opened, synchronize, reopened] -permissions: - pull-requests: write - contents: write jobs: - stub_job_name: - name: Stub Job + bump: + permissions: + contents: write + pull-requests: write runs-on: ubuntu-latest steps: - - name: Stub step - run: | - echo "Stub to iterate via PR" - \ No newline at end of file + - uses: actions/checkout@v5 + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: apply --config .ci/updatecli/bump-java-version.yml --values .ci/updatecli/values.d/scm.yml + version-file: .updatecli-version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LOGSTASH_BRANCH: ${{ github.ref_name }} diff --git a/.github/workflows/bump-logstash.yml b/.github/workflows/bump-logstash.yml new file mode 100644 index 0000000000..6679f1ed61 --- /dev/null +++ b/.github/workflows/bump-logstash.yml @@ -0,0 +1,31 @@ +name: bump-logstash-version + +on: + workflow_dispatch: + inputs: + logstash_version: + description: 'Logstash version (example: 9.1.4)' + required: true + type: string + logstash_branch: + description: 'Logstash branch (example: 9.1)' + required: true + type: string + +jobs: + bump: + permissions: + contents: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: apply --config .ci/updatecli/bump-logstash-version.yml --values .ci/updatecli/values.d/scm.yml + version-file: .updatecli-version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LOGSTASH_BRANCH: "${{ github.event.inputs.logstash_branch }}" + LOGSTASH_VERSION: "${{ github.event.inputs.logstash_version }}" diff --git a/.github/workflows/critical_vulnerability_scan.yml b/.github/workflows/critical_vulnerability_scan.yml index a500c9448e..dc437bb41c 100644 --- a/.github/workflows/critical_vulnerability_scan.yml +++ b/.github/workflows/critical_vulnerability_scan.yml @@ -17,7 +17,7 @@ jobs: - run: tar -zxf ../build/logstash-*.tar.gz working-directory: ./scan - name: scan image - uses: anchore/scan-action@f6601287cdb1efc985d6b765bbf99cb4c0ac29d8 # v7.0.0 + uses: anchore/scan-action@a5605eb0943e46279cb4fbd9d44297355d3520ab # v7.0.2 with: path: "./scan" fail-build: true diff --git a/.github/workflows/lint_docs.yml b/.github/workflows/lint_docs.yml index f1620d0d69..c70b98fe69 100644 --- a/.github/workflows/lint_docs.yml +++ b/.github/workflows/lint_docs.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v5 - name: Set up Node.js - uses: actions/setup-node@v5 + uses: actions/setup-node@v6 with: node-version: 16.13.2 cache: npm diff --git a/.github/workflows/logstash_project_board.yml b/.github/workflows/logstash_project_board.yml index 48858610f5..7a62e6416d 100644 --- a/.github/workflows/logstash_project_board.yml +++ b/.github/workflows/logstash_project_board.yml @@ -18,6 +18,6 @@ jobs: clientMutationId } } - projectid: "PVT_kwDOAGc3Zs0SEg" + projectid: "PVT_kwDOAGc3Zs4AMlnl" contentid: ${{ github.event.issue.node_id }} GITHUB_TOKEN: ${{ secrets.PROJECT_TOKEN }} diff --git a/.updatecli-version b/.updatecli-version new file mode 100644 index 0000000000..5c5d557118 --- /dev/null +++ b/.updatecli-version @@ -0,0 +1 @@ +v0.104.0 \ No newline at end of file diff --git a/build.gradle b/build.gradle index b6987700c4..27409b0cb0 100644 --- a/build.gradle +++ b/build.gradle @@ -90,6 +90,8 @@ allprojects { tasks.withType(Test) { // Add Exports to enable tests to run in JDK17 jvmArgs = [ + "-Djruby.compile.invokedynamic=true", + "-Dlog4j2.isThreadContextMapInheritable=true", "--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", @@ -748,23 +750,6 @@ class JDKDetails { return createElasticCatalogDownloadUrl() } - // throws an error iff local version in versions.yml doesn't match the latest from JVM catalog. - void checkLocalVersionMatchingLatest() { - // retrieve the metadata from remote - def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${major}_${osName}" - def catalogMetadataUrl = URI.create(url).toURL() - def catalogConnection = catalogMetadataUrl.openConnection() - catalogConnection.requestMethod = 'GET' - assert catalogConnection.responseCode == 200 - - def metadataRetrieved = catalogConnection.content.text - def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved) - - if (catalogMetadata.version != revision || catalogMetadata.revision != build) { - throw new GradleException("Found new jdk version. Please update version.yml to ${catalogMetadata.version} build ${catalogMetadata.revision}") - } - } - private String createElasticCatalogDownloadUrl() { // Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK @@ -874,13 +859,6 @@ tasks.register("downloadJdk", Download) { } } -tasks.register("checkNewJdkVersion") { - // use Linux x86_64 as canary platform - def jdkDetails = new JDKDetails(gradle.ext.versions.bundled_jdk, "linux", "x86_64") - // throws Gradle exception if local and remote doesn't match - jdkDetails.checkLocalVersionMatchingLatest() -} - tasks.register("deleteLocalJdk", Delete) { // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] String osName = selectOsType() diff --git a/catalog-info.yaml b/catalog-info.yaml index 143acb634b..27350e8ef7 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -33,7 +33,6 @@ spec: - resource:logstash-windows-jdk-matrix-pipeline - resource:logstash-benchmark-pipeline - resource:logstash-health-report-tests-pipeline - - resource:logstash-jdk-availability-check-pipeline # *********************************** # Declare serverless IT pipeline @@ -479,6 +478,57 @@ spec: # SECTION END: Exhaustive tests pipeline # ************************************** +# **************************************** +# SECTION START: Smart exhaustive tests pipeline +# **************************************** + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-smart-exhaustive-tests-pipeline + description: 'Logstash Smart Exhaustive tests pipeline' + links: + - title: 'Logstash Smart Exhaustive tests pipeline' + url: https://buildkite.com/elastic/logstash-smart-exhaustive-tests-pipeline +spec: + type: buildkite-pipeline + owner: group:logstash + system: platform-ingest + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: "Logstash Smart Exhaustive tests pipeline" + description: '🔍 Run smart exhaustive tests against Logstash using different operating systems' + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/smart_exhaustive_tests_pipeline.yml" + provider_settings: + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_branches: false + build_tags: false + filter_enabled: true + filter_condition: >- + build.creator.name == 'elasticmachine' && build.pull_request.id != null + cancel_intermediate_builds: true + skip_intermediate_builds: true + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +# ************************************** +# SECTION END: Smart exhaustive tests pipeline +# ************************************** + # ******************************************** # Declare supported plugin tests pipeline # ******************************************** @@ -744,62 +794,3 @@ spec: branch: main cronline: 30 20 * * * message: Daily trigger of Health Report Tests Pipeline - -# ******************************* -# SECTION END: Health Report Tests pipeline -# ******************************* - -# *********************************** -# Declare JDK check pipeline -# *********************************** ---- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json -apiVersion: backstage.io/v1alpha1 -kind: Resource -metadata: - name: logstash-jdk-availability-check-pipeline - description: ":logstash: check availability of new JDK version" -spec: - type: buildkite-pipeline - owner: group:logstash - system: platform-ingest - implementation: - apiVersion: buildkite.elastic.dev/v1 - kind: Pipeline - metadata: - name: logstash-jdk-availability-check-pipeline - spec: - repository: elastic/logstash - pipeline_file: ".buildkite/jdk_availability_check_pipeline.yml" - maximum_timeout_in_minutes: 10 - provider_settings: - trigger_mode: none # don't trigger jobs from github activity - env: - ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' - SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' - SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' - SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' - teams: - logstash: - access_level: MANAGE_BUILD_AND_READ - ingest-eng-prod: - access_level: MANAGE_BUILD_AND_READ - everyone: - access_level: READ_ONLY - schedules: - Weekly JDK availability check (main): - branch: main - cronline: 0 2 * * 1 # every Monday@2AM UTC - message: Weekly trigger of JDK update availability pipeline per branch - env: - PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline' - Weekly JDK availability check (8.19): - branch: "8.19" - cronline: 0 2 * * 1 # every Monday@2AM UTC - message: Weekly trigger of JDK update availability pipeline per branch - env: - PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline' - -# ******************************* -# SECTION END: JDK check pipeline -# ******************************* diff --git a/ci/check_jdk_version_availability.sh b/ci/check_jdk_version_availability.sh deleted file mode 100755 index 2ce40dc7b2..0000000000 --- a/ci/check_jdk_version_availability.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" - -echo "Checking local JDK version against latest remote from JVM catalog" -./gradlew checkNewJdkVersion \ No newline at end of file diff --git a/ci/docker_acceptance_tests.sh b/ci/docker_acceptance_tests.sh index b1e62de2c2..74df2cc102 100755 --- a/ci/docker_acceptance_tests.sh +++ b/ci/docker_acceptance_tests.sh @@ -8,6 +8,9 @@ set -x export JRUBY_OPTS="-J-Xmx1g" export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" +# Use local artifacts for acceptance test Docker builds +export LOCAL_ARTIFACTS=true + if [ -n "$BUILD_JAVA_HOME" ]; then GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" fi @@ -48,7 +51,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then elif [[ $SELECTED_TEST_SUITE == "full" ]]; then echo "--- Building $SELECTED_TEST_SUITE docker images" cd $LS_HOME - rake artifact:build_docker_full + rake artifact:docker echo "--- Acceptance: Installing dependencies" cd $QA_DIR bundle install diff --git a/ci/unit_tests.sh b/ci/unit_tests.sh index 82a670b071..d7a760d693 100755 --- a/ci/unit_tests.sh +++ b/ci/unit_tests.sh @@ -10,8 +10,6 @@ export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false export SPEC_OPTS="--order rand --format documentation" export CI=true export TEST_DEBUG=true -# don't rely on bash booleans for truth checks, since some CI platforms don't have a way to specify env vars as boolean -export ENABLE_SONARQUBE=${ENABLE_SONARQUBE:-"true"} if [ -n "$BUILD_JAVA_HOME" ]; then GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" @@ -19,15 +17,9 @@ fi SELECTED_TEST_SUITE=$1 -SONAR_ARGS=() -if [[ $(echo $ENABLE_SONARQUBE | tr '[:lower:]' '[:upper:]') == "TRUE" ]]; then - SONAR_ARGS=("jacocoTestReport") - export COVERAGE=true -fi - if [[ $SELECTED_TEST_SUITE == $"java" ]]; then echo "Running Java Tests" - ./gradlew javaTests "${SONAR_ARGS[@]}" --console=plain --warning-mode all + ./gradlew javaTests --console=plain --warning-mode all elif [[ $SELECTED_TEST_SUITE == $"ruby" ]]; then echo "Running Ruby unit tests" ./gradlew rubyTests --console=plain --warning-mode all diff --git a/config/jvm.options b/config/jvm.options index 084ff034b6..f339bff690 100644 --- a/config/jvm.options +++ b/config/jvm.options @@ -41,9 +41,6 @@ # use our provided JNA always versus the system one #-Djna.nosys=true -# Turn on JRuby invokedynamic --Djruby.compile.invokedynamic=true - ## heap dumps # generate a heap dump when an allocation from the Java heap fails @@ -60,9 +57,6 @@ # Entropy source for randomness -Djava.security.egd=file:/dev/urandom -# Copy the logging context from parent threads to children --Dlog4j2.isThreadContextMapInheritable=true - # FasterXML/jackson defaults # # Sets the maximum string length (in chars or bytes, depending on input context). diff --git a/config/logstash.yml b/config/logstash.yml index e4d008ca5c..3774387562 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -49,6 +49,12 @@ # # pipeline.batch.delay: 50 # +# Set the pipeline's batch metrics reporting mode. It can be "disabled" to disable it. +# "minimal" to collect only 1% of the batches metrics, "full" to collect all batches. +# Default is "minimal". +# +# pipeline.batch.metrics.sampling_mode: "minimal" +# # Force Logstash to exit during shutdown even if there are still inflight # events in memory. By default, logstash will refuse to quit until all # received events have been pushed to the outputs. @@ -223,6 +229,10 @@ # # queue.checkpoint.writes: 1024 # +# If using queue.type: persisted, the compression goal. Valid values are `none`, `speed`, `balanced`, and `size`. +# The default `none` is able to decompress previously-written events, even if they were compressed. +# +# queue.compression: none # # ------------ Dead-Letter Queue Settings -------------- # Flag to turn on dead-letter queue. diff --git a/docker/Makefile b/docker/Makefile index cdc9915b6b..c220f57f89 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -132,19 +132,12 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="full" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \ tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_full: public-dockerfiles_full - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_full && cd dockerfile_build_full && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) . - public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -153,19 +146,12 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="oss" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \ tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_oss: public-dockerfiles_oss - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_oss && cd dockerfile_build_oss && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-oss:$(VERSION_TAG) . - public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -174,19 +160,12 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="wolfi" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \ tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_wolfi: public-dockerfiles_wolfi - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) . - public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -195,19 +174,12 @@ public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(CO version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="observability-sre" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-observability-sre" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \ tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_observability-sre: public-dockerfiles_observability-sre - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_observability-sre && cd dockerfile_build_observability-sre && \ - tar -zxf ../../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-observability-sre:$(VERSION_TAG) . - public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ elastic_version="${ELASTIC_VERSION}" \ @@ -219,7 +191,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="ironbank" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \ cd $(ARTIFACTS_DIR)/ironbank && \ cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \ diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index 95fc569b23..d1e976bbab 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -58,6 +58,7 @@ var validSettings = []string{ "queue.checkpoint.acks", "queue.checkpoint.writes", "queue.checkpoint.interval", // remove it for #17155 + "queue.compression", "queue.drain", "dead_letter_queue.enable", "dead_letter_queue.max_bytes", diff --git a/docs/reference/configuring-centralized-pipelines.md b/docs/reference/configuring-centralized-pipelines.md index fc56a4a4f9..04ec13dd65 100644 --- a/docs/reference/configuring-centralized-pipelines.md +++ b/docs/reference/configuring-centralized-pipelines.md @@ -144,7 +144,7 @@ This setting can be used only if `xpack.management.elasticsearch.ssl.certificate ## Wildcard support in pipeline ID [wildcard-in-pipeline-id] -Pipeline IDs must begin with a letter or underscore and contain only letters, underscores, dashes, and numbers. You can use `*` in `xpack.management.pipeline.id` to match any number of letters, underscores, dashes, and numbers. +Pipeline IDs must begin with a letter or underscore and contain only letters, underscores, dashes, hyphens and numbers. You can use `*` in `xpack.management.pipeline.id` to match any number of letters, underscores, dashes, hyphens, and numbers. ```shell xpack.management.pipeline.id: ["*logs", "*apache*", "tomcat_log"] diff --git a/docs/reference/connecting-to-serverless.md b/docs/reference/connecting-to-serverless.md index 8e381b63b4..aeccb03ccc 100644 --- a/docs/reference/connecting-to-serverless.md +++ b/docs/reference/connecting-to-serverless.md @@ -22,7 +22,7 @@ Set the value to port :443 instead. :::: -## Communication between {{ls}} {{es-serverless}} [connecting-to-elasticsearch-serverless] +## Communication between {{ls}} and {{es-serverless}} [connecting-to-elasticsearch-serverless] [{{es-serverless}}](docs-content://solutions/search/serverless-elasticsearch-get-started.md) simplifies safe, secure communication between {{ls}} and {{es}}. When you configure the Elasticsearch output plugin to use [`cloud_id`](logstash-docs-md://lsr/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-cloud_id) and an [`api_key`](logstash-docs-md://lsr/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-api_key), no additional SSL configuration is needed. diff --git a/docs/reference/logstash-settings-file.md b/docs/reference/logstash-settings-file.md index 7bcce3c853..197e7e5505 100644 --- a/docs/reference/logstash-settings-file.md +++ b/docs/reference/logstash-settings-file.md @@ -48,6 +48,7 @@ The `logstash.yml` file includes these settings. | `pipeline.workers` | The number of workers that will, in parallel, execute the filter and outputstages of the pipeline. This setting uses the[`java.lang.Runtime.getRuntime.availableProcessors`](https://docs.oracle.com/javase/7/docs/api/java/lang/Runtime.md#availableProcessors())value as a default if not overridden by `pipeline.workers` in `pipelines.yml` or`pipeline.workers` from `logstash.yml`. If you have modified this setting andsee that events are backing up, or that the CPU is not saturated, considerincreasing this number to better utilize machine processing power. | Number of the host’s CPU cores | | `pipeline.batch.size` | The maximum number of events an individual worker thread will collect from inputs before attempting to execute its filters and outputs. Larger batch sizes are generally more efficient, but come at the cost of increased memory overhead. You may need to increase JVM heap space in the `jvm.options` config file. See [Logstash Configuration Files](/reference/config-setting-files.md) for more info. | `125` | | `pipeline.batch.delay` | When creating pipeline event batches, how long in milliseconds to wait for each event before dispatching an undersized batch to pipeline workers. | `50` | +| `pipeline.batch.metrics.sampling_mode` {applies_to}`stack: preview 9.2.0`| Controls frequency of collection of batch size metrics. These metrics measure the actual number of events and byte size of batches processed through a pipeline. This can be helpful to tune `pipeline.batch.size` to reflect the actual batch sizes processed.

Note: This feature is in **technical preview** and may change in the future.

Current options are:

* `disabled`: disabling the collection.
* `minimal`: calculate based on a subset of batches.(default)
* `full`: calculate based on every processed batch.
| `minimal` | | `pipeline.unsafe_shutdown` | When set to `true`, forces Logstash to exit during shutdown even if there are still inflight events in memory. By default, Logstash will refuse to quit until all received events have been pushed to the outputs. Enabling this option can lead to data loss during shutdown. | `false` | | `pipeline.plugin_classloaders` | (Beta) Load Java plugins in independent classloaders to isolate their dependencies. | `false` | | `pipeline.ordered` | Set the pipeline event ordering. Valid options are:

* `auto`. Automatically enables ordering if the `pipeline.workers` setting is `1`, and disables otherwise.
* `true`. Enforces ordering on the pipeline and prevents Logstash from starting if there are multiple workers.
* `false`. Disables the processing required to preserve order. Ordering will not be guaranteed, but you save the processing cost of preserving order.
| `auto` | @@ -68,6 +69,7 @@ The `logstash.yml` file includes these settings. | `queue.checkpoint.acks` | The maximum number of ACKed events before forcing a checkpoint when persistent queues are enabled (`queue.type: persisted`). Specify `queue.checkpoint.acks: 0` to set this value to unlimited. | 1024 | | `queue.checkpoint.writes` | The maximum number of written events before forcing a checkpoint when persistent queues are enabled (`queue.type: persisted`). Specify `queue.checkpoint.writes: 0` to set this value to unlimited. | 1024 | | `queue.checkpoint.retry` | When enabled, Logstash will retry four times per attempted checkpoint write for any checkpoint writes that fail. Any subsequent errors are not retried. This is a workaround for failed checkpoint writes that have been seen only on Windows platform, filesystems with non-standard behavior such as SANs and is not recommended except in those specific circumstances. (`queue.type: persisted`) | `true` | +| `queue.compression` {applies_to}`stack: ga 9.2` | Set a persisted queue compression level, which allows the pipeline to reduce the event size on disk at the cost of CPU usage. Possible values are `speed`, `balanced`, and `size`. | `none` | | `queue.drain` | When enabled, Logstash waits until the persistent queue (`queue.type: persisted`) is drained before shutting down. | `false` | | `dead_letter_queue.enable` | Flag to instruct Logstash to enable the DLQ feature supported by plugins. | `false` | | `dead_letter_queue.max_bytes` | The maximum size of each dead letter queue. Entries will be dropped if they would increase the size of the dead letter queue beyond this setting. | `1024mb` | diff --git a/docs/reference/persistent-queues.md b/docs/reference/persistent-queues.md index 8c2bed6639..b10c117981 100644 --- a/docs/reference/persistent-queues.md +++ b/docs/reference/persistent-queues.md @@ -84,6 +84,17 @@ If you want to define values for a specific pipeline, use [`pipelines.yml`](/ref `queue.checkpoint.interval` {applies_to}`stack: deprecated 9.1` : Sets the interval in milliseconds when a checkpoint is forced on the head page. Default is `1000`. Set to `0` to eliminate periodic checkpoints. +`queue.compression` {applies_to}`stack: ga 9.2` +: Sets the event compression level for use with the Persisted Queue. Default is `none`. Possible values are: + * `none`: does not perform compression, but reads compressed events + * `speed`: optimize for fastest compression operation + * `size`: optimize for smallest possible size on disk, spending more CPU + * `balanced`: a balance between the `speed` and `size` settings +:::{important} +Compression can be enabled for an existing PQ, but once compressed elements have been added to a PQ, that PQ cannot be read by previous Logstash releases that did not support compression. +If you need to downgrade Logstash after enabling the PQ, you will need to either delete the PQ or run the pipeline with `queue.drain: true` first to ensure that no compressed elements remain. +::: + ## Configuration notes [pq-config-notes] Every situation and environment is different, and the "ideal" configuration varies. If you optimize for performance, you may increase your risk of losing data. If you optimize for data protection, you may impact performance. diff --git a/docs/reference/tips-best-practices.md b/docs/reference/tips-best-practices.md index f6003263f5..2d2eed478d 100644 --- a/docs/reference/tips-best-practices.md +++ b/docs/reference/tips-best-practices.md @@ -60,13 +60,15 @@ filter { # we use a "temporal" field with a predefined arbitrary known value that # lives only in filtering stage. add_field => { "[@metadata][test_field_check]" => "a null value" } + } +filter { + mutate { # we copy the field of interest into that temporal field. # If the field doesn't exist, copy is not executed. copy => { "test_field" => "[@metadata][test_field_check]" } } - # now we now if testField didn't exists, our field will have # the initial arbitrary value if [@metadata][test_field_check] == "a null value" { diff --git a/docs/reference/tuning-logstash.md b/docs/reference/tuning-logstash.md index d7958a026e..0416550544 100644 --- a/docs/reference/tuning-logstash.md +++ b/docs/reference/tuning-logstash.md @@ -50,6 +50,7 @@ Make sure you’ve read the [Performance troubleshooting](/reference/performance If you plan to modify the default pipeline settings, take into account the following suggestions: * The total number of inflight events is determined by the product of the `pipeline.workers` and `pipeline.batch.size` settings. This product is referred to as the *inflight count*. Keep the value of the inflight count in mind as you adjust the `pipeline.workers` and `pipeline.batch.size` settings. Pipelines that intermittently receive large events at irregular intervals require sufficient memory to handle these spikes. Set the JVM heap space accordingly in the `jvm.options` config file (See [Logstash Configuration Files](/reference/config-setting-files.md) for more info). +* {applies_to}`stack: preview 9.2.0` Consider enabling the metering of batch sizes using the setting `pipeline.batch.metrics.sampling_mode` to help you understand the actual batch sizes being processed by your pipeline. This setting can be useful tuning the `pipeline.batch.size` setting. For more details see [logstash.yml](/reference/logstash-settings-file.md). * Measure each change to make sure it increases, rather than decreases, performance. * Ensure that you leave enough memory available to cope with a sudden increase in event size. For example, an application that generates exceptions that are represented as large blobs of text. * The number of workers may be set higher than the number of CPU cores since outputs often spend idle time in I/O wait conditions. diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 38d7633bf7..b9d8ee463a 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -21,6 +21,28 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [logstash-next-fixes] % * +## 9.1.5 [logstash-9.1.5-release-notes] + +No user-facing changes in Logstash core. + +### Plugins [logstash-plugin-9.1.5-changes] + +**Elasticsearch Filter - 4.3.1** + +* Added support for encoded and non-encoded api-key formats on plugin configuration [#203](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/203) + +**Elasticsearch Input - 5.2.1** + +* Added support for encoded and non-encoded api-key formats on plugin configuration [#237](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/237) + +**Jdbc Integration - 5.6.1** + +* Fixes an issue where the `jdbc_static` filter's throughput was artificially limited to 4 concurrent queries, causing the plugin to become a bottleneck in pipelines with more than 4 workers. Each instance of the plugin is now limited to 16 concurrent queries, with increased timeouts to eliminate enrichment failures. [#187](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/187) + +**Elasticsearch Output - 12.0.7** + +* Support both, encoded and non encoded api-key formats on plugin configuration [#1223](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1223) + ## 9.1.4 [logstash-9.1.4-release-notes] ### Features and enhancements [logstash-9.1.4-features-enhancements] @@ -31,7 +53,7 @@ To check for security updates, go to [Security announcements for the Elastic sta ### Fixes [logstash-9.1.4-fixes] -* Gauge type metrics, such as current and peak connection counts of Elastic Agent, are now available in the `_node/stats` API response when the `vertices=true` parameter is included. These metrics are particularly useful for monitoring {ls} plugin activity on the {ls} Integration dashboards [#18090](https://github.com/elastic/logstash/pull/18090) +* Gauge type metrics, such as current and peak connection counts of Elastic Agent, are now available in the `_node/stats` API response when the `vertices=true` parameter is included. These metrics are particularly useful for monitoring {{ls}} plugin activity on the {{ls}} Integration dashboards [#18090](https://github.com/elastic/logstash/pull/18090) * Improve logstash release artifacts file metadata: mtime is preserved when buiilding tar archives [#18091](https://github.com/elastic/logstash/pull/18091) @@ -177,13 +199,23 @@ The Elasticsearch Input now provides [support](https://github.com/logstash-plugi **Tcp Output - 7.0.1** * Call connection check after connect [#61](https://github.com/logstash-plugins/logstash-output-tcp/pull/61) +## 9.0.8 [logstash-9.0.8-release-notes] + +No user-facing changes in Logstash core. + +### Plugins [logstash-plugin-9.0.8-changes] + +**Elasticsearch Output - 12.0.7** + +* Support both, encoded and non-encoded api-key formats on plugin configuration [#1223](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1223) + ## 9.0.7 [logstash-9.0.7-release-notes] ### Fixes [logstash-9.0.7-fixes] -* Gauge type metrics, such as current and peak connection counts of Elastic Agent, are now available in the `_node/stats` API response when the `vertices=true` parameter is included. These metrics are particularly useful for monitoring {ls} plugin activity on the {ls} Integration dashboards. [#18089](https://github.com/elastic/logstash/pull/18089) -* Improve logstash release artifacts file metadata: mtime is preserved when buiilding tar archives. [#18111](https://github.com/elastic/logstash/pull/18111) +* Gauge type metrics, such as current and peak connection counts of Elastic Agent, are now available in the `_node/stats` API response when the `vertices=true` parameter is included. These metrics are particularly useful for monitoring {{ls}} plugin activity on the {{ls}} Integration dashboards. [#18089](https://github.com/elastic/logstash/pull/18089) +* Improve logstash release artifacts file metadata: mtime is preserved when building tar archives. [#18111](https://github.com/elastic/logstash/pull/18111) ### Plugins [logstash-plugin-9.0.7-changes] diff --git a/docs/static/spec/openapi/logstash-api.yaml b/docs/static/spec/openapi/logstash-api.yaml index b979006b7d..cbf133cc40 100644 --- a/docs/static/spec/openapi/logstash-api.yaml +++ b/docs/static/spec/openapi/logstash-api.yaml @@ -810,6 +810,7 @@ paths: - stats for each configured filter or output stage - info about config reload successes and failures (when [config reload](https://www.elastic.co/guide/en/logstash/current/reloading-config.html) is enabled) - info about the persistent queue (when [persistent queues](https://www.elastic.co/guide/en/logstash/current/persistent-queues.html) are enabled) + - metrics related to processed batch sizes. Includes the size in bytes and the number of events of batches processed in this pipeline. (when setting [pipeline.batch.metrics.sampling_mode](https://www.elastic.co/docs/reference/logstash/logstash-settings-file.html) is not `disabled`). content: application/json: @@ -821,6 +822,15 @@ paths: example: pipelines: beats-es: + batch: + event_count: + current: 78 + average: + lifetime: 115 + byte_size: + current: 32767 + average: + lifetime: 14820 events: duration_in_millis: 365495 in: 216610 @@ -1095,6 +1105,13 @@ paths: value: pipelines: heartbeat-ruby-stdout: + batch: + event_count: + average: + lifetime: 115 + byte_size: + average: + lifetime: 14820 events: queue_push_duration_in_millis: 159 in: 45 @@ -2340,6 +2357,64 @@ components: max_queue_size_in_bytes: type: integer format: int64 + compression: + type: object + properties: + encode: + type: object + properties: + goal: + - enum: + - speed + - balanced + - size + ratio: + type: object + description: the ratio of event size in bytes to its representation on disk + properties: + lifetime: + oneOf: + - type: number + - enum: + - "Infinity" + - "NaN" + - "-Infinity" + spend: + type: object + description: the fraction of wall-clock time spent encoding events + properties: + lifetime: + oneOf: + - type: number + - enum: + - "Infinity" + - "NaN" + - "-Infinity" + decode: + type: object + properties: + ratio: + type: object + description: the ratio of event representation on disk to event size + properties: + lifetime: + oneOf: + - type: number + - enum: + - "Infinity" + - "NaN" + - "-Infinity" + spend: + type: object + description: the fraction of wall-clock time spent decoding events + properties: + lifetime: + oneOf: + - type: number + - enum: + - "Infinity" + - "NaN" + - "-Infinity" - type: object description: "The metrics of memory queue." required: diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 1b33c55baa..a4b76b9530 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index d4081da476..e2847c8200 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 23d15a9367..f5feea6d6b 100755 --- a/gradlew +++ b/gradlew @@ -86,7 +86,8 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -114,7 +115,7 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH="\\\"\\\"" +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. @@ -205,7 +206,7 @@ fi DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. @@ -213,7 +214,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ - -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + org.gradle.wrapper.GradleWrapperMain \ "$@" # Stop when "xargs" is not available. diff --git a/gradlew.bat b/gradlew.bat index 5eed7ee845..9b42019c79 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -70,11 +70,11 @@ goto fail :execute @rem Setup the command line -set CLASSPATH= +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 5c0db0f5a3..de201dc667 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -23,22 +23,14 @@ buildscript { plugins { id "jacoco" - id "org.sonarqube" version "4.3.0.3225" } apply plugin: 'jacoco' -apply plugin: "org.sonarqube" repositories { mavenCentral() } -sonarqube { - properties { - property 'sonar.coverage.jacoco.xmlReportPaths', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml" - } -} - jacoco { toolVersion = "0.8.9" } @@ -239,6 +231,7 @@ dependencies { implementation 'commons-codec:commons-codec:1.17.0' // transitively required by httpclient // Jackson version moved to versions.yml in the project root (the JrJackson version is there too) implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" + implementation "com.github.luben:zstd-jni:1.5.7-4" api "com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}" api "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" implementation 'org.codehaus.janino:janino:3.1.0' diff --git a/logstash-core/lib/logstash/api/commands/stats.rb b/logstash-core/lib/logstash/api/commands/stats.rb index 5bf1b3e3a0..7ae782fadf 100644 --- a/logstash-core/lib/logstash/api/commands/stats.rb +++ b/logstash-core/lib/logstash/api/commands/stats.rb @@ -172,6 +172,29 @@ def plugin_stats(stats, plugin_type) end end + def refine_batch_metrics(stats) + # current is a tuple of [event_count, byte_size] store the reference locally to avoid repeatedly + # reading and retrieve unrelated values + current_data_point = stats[:batch][:current] + { + :event_count => { + # current_data_point is an instance of org.logstash.instrument.metrics.gauge.LazyDelegatingGauge so need to invoke getValue() to obtain the actual value + :current => current_data_point.value[0], + :average => { + # average return a FlowMetric which and we need to invoke getValue to obtain the map with metric details. + :lifetime => stats[:batch][:event_count][:average].value["lifetime"] ? stats[:batch][:event_count][:average].value["lifetime"].round : 0 + } + }, + :byte_size => { + :current => current_data_point.value[1], + :average => { + :lifetime => stats[:batch][:byte_size][:average].value["lifetime"] ? stats[:batch][:byte_size][:average].value["lifetime"].round : 0 + } + } + } + end + private :refine_batch_metrics + def report(stats, extended_stats = nil, opts = {}) ret = { :events => stats[:events], @@ -190,6 +213,7 @@ def report(stats, extended_stats = nil, opts = {}) :batch_delay => stats.dig(:config, :batch_delay), } } + ret[:batch] = refine_batch_metrics(stats) if stats.include?(:batch) ret[:dead_letter_queue] = stats[:dlq] if stats.include?(:dlq) # if extended_stats were provided, enrich the return value diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index d49d3d9b6f..4eb8de501d 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -75,7 +75,7 @@ def self.as_java_range(r) Setting::StringSetting.new("api.environment", "production"), Setting::StringSetting.new("api.auth.type", "none", true, %w(none basic)), Setting::StringSetting.new("api.auth.basic.username", nil, false).nullable, - Setting::Password.new("api.auth.basic.password", nil, false).nullable, + Setting::PasswordSetting.new("api.auth.basic.password", nil, false).nullable, Setting::StringSetting.new("api.auth.basic.password_policy.mode", "WARN", true, %w[WARN ERROR]), Setting::NumericSetting.new("api.auth.basic.password_policy.length.minimum", 8), Setting::StringSetting.new("api.auth.basic.password_policy.include.upper", "REQUIRED", true, %w[REQUIRED OPTIONAL]), @@ -84,8 +84,9 @@ def self.as_java_range(r) Setting::StringSetting.new("api.auth.basic.password_policy.include.symbol", "OPTIONAL", true, %w[REQUIRED OPTIONAL]), Setting::BooleanSetting.new("api.ssl.enabled", false), Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable, - Setting::Password.new("api.ssl.keystore.password", nil, false).nullable, + Setting::PasswordSetting.new("api.ssl.keystore.password", nil, false).nullable, Setting::StringArray.new("api.ssl.supported_protocols", nil, true, %w[TLSv1 TLSv1.1 TLSv1.2 TLSv1.3]), + Setting::StringSetting.new("pipeline.batch.metrics.sampling_mode", "minimal", true, ["disabled", "minimal", "full"]), Setting::StringSetting.new("queue.type", "memory", true, ["persisted", "memory"]), Setting::BooleanSetting.new("queue.drain", false), Setting::Bytes.new("queue.page_capacity", "64mb"), @@ -95,6 +96,7 @@ def self.as_java_range(r) Setting::NumericSetting.new("queue.checkpoint.writes", 1024), # 0 is unlimited Setting::NumericSetting.new("queue.checkpoint.interval", 1000), # remove it for #17155 Setting::BooleanSetting.new("queue.checkpoint.retry", true), + Setting::StringSetting.new("queue.compression", "none", true, %w(none speed balanced size disabled)), Setting::BooleanSetting.new("dead_letter_queue.enable", false), Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"), Setting::NumericSetting.new("dead_letter_queue.flush_interval", 5000), diff --git a/logstash-core/lib/logstash/instrument/collector.rb b/logstash-core/lib/logstash/instrument/collector.rb index 1a467f3cce..a1a7fc3c07 100644 --- a/logstash-core/lib/logstash/instrument/collector.rb +++ b/logstash-core/lib/logstash/instrument/collector.rb @@ -74,6 +74,12 @@ def get(namespaces_path, key, type) end end + ## + # @return [Metric]: the metric that exists after registration + def register(namespaces_path, key, &metric_supplier) + @metric_store.fetch_or_store(namespaces_path, key, &metric_supplier) + end + # test injection, see MetricExtFactory def initialize_metric(type, namespaces_path, key) MetricType.create(type, namespaces_path, key) diff --git a/logstash-core/lib/logstash/java_pipeline.rb b/logstash-core/lib/logstash/java_pipeline.rb index 11ce715bea..600b689d0f 100644 --- a/logstash-core/lib/logstash/java_pipeline.rb +++ b/logstash-core/lib/logstash/java_pipeline.rb @@ -267,6 +267,7 @@ def start_workers @preserve_event_order = preserve_event_order?(pipeline_workers) batch_size = settings.get("pipeline.batch.size") batch_delay = settings.get("pipeline.batch.delay") + batch_metric_sampling = settings.get("pipeline.batch.metrics.sampling_mode") max_inflight = batch_size * pipeline_workers @@ -287,6 +288,7 @@ def start_workers "pipeline.batch.size" => batch_size, "pipeline.batch.delay" => batch_delay, "pipeline.max_inflight" => max_inflight, + "batch_metric_sampling" => batch_metric_sampling, "pipeline.sources" => pipeline_source_details) @logger.info("Starting pipeline", pipeline_log_params) diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index c23455df93..39fe4fa940 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -58,6 +58,7 @@ def self.included(base) "path.dead_letter_queue", "path.queue", "pipeline.batch.delay", + "pipeline.batch.metrics.sampling_mode", "pipeline.batch.size", "pipeline.id", "pipeline.reloadable", @@ -69,6 +70,7 @@ def self.included(base) "queue.checkpoint.interval", # remove it for #17155 "queue.checkpoint.writes", "queue.checkpoint.retry", + "queue.compression", "queue.drain", "queue.max_bytes", "queue.max_events", @@ -437,27 +439,9 @@ def validate(value) java_import org.logstash.settings.NullableStringSetting - class Password < Coercible - def initialize(name, default = nil, strict = true) - super(name, LogStash::Util::Password, default, strict) - end - - def coerce(value) - return value if value.kind_of?(LogStash::Util::Password) - - if value && !value.kind_of?(::String) - raise(ArgumentError, "Setting `#{name}` could not coerce non-string value to password") - end - - LogStash::Util::Password.new(value) - end - - def validate(value) - super(value) - end - end + java_import org.logstash.settings.PasswordSetting - class ValidatedPassword < Setting::Password + class ValidatedPassword < Setting::PasswordSetting def initialize(name, value, password_policies) @password_policies = password_policies super(name, value, true) diff --git a/logstash-core/spec/logstash/acked_queue_concurrent_stress_spec.rb b/logstash-core/spec/logstash/acked_queue_concurrent_stress_spec.rb index ccd37f6bb4..c7c1eae328 100644 --- a/logstash-core/spec/logstash/acked_queue_concurrent_stress_spec.rb +++ b/logstash-core/spec/logstash/acked_queue_concurrent_stress_spec.rb @@ -38,7 +38,7 @@ end let(:queue) do - described_class.new(queue_settings) + described_class.new(queue_settings, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED) end let(:writer_threads) do diff --git a/logstash-core/spec/logstash/api/modules/node_stats_spec.rb b/logstash-core/spec/logstash/api/modules/node_stats_spec.rb index 48b73f80ce..136c1349db 100644 --- a/logstash-core/spec/logstash/api/modules/node_stats_spec.rb +++ b/logstash-core/spec/logstash/api/modules/node_stats_spec.rb @@ -21,8 +21,13 @@ require "logstash/api/modules/node_stats" describe LogStash::Api::Modules::NodeStats do - # enable PQ to ensure PQ-related metrics are present - include_context "api setup", {"queue.type" => "persisted"} + + include_context "api setup", { + # enable PQ to ensure PQ-related metrics are present + "queue.type" => "persisted", + #enable batch metrics + "pipeline.batch.metrics.sampling_mode" => "full" + } include_examples "not found" extend ResourceDSLMethods @@ -142,6 +147,20 @@ "path" => String, "free_space_in_bytes" => Numeric } + }, + "batch" => { + "event_count" => { + "current" => Numeric, + "average" => { + "lifetime" => Numeric + } + }, + "byte_size" => { + "current" => Numeric, + "average" => { + "lifetime" => Numeric + } + } } } }, diff --git a/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb b/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb index 1c1f9a1fb2..3fa5bf325a 100644 --- a/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb +++ b/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb @@ -113,7 +113,7 @@ def threaded_read_client end context "WrappedSynchronousQueue" do - let(:queue) { LogStash::WrappedSynchronousQueue.new(1024) } + let(:queue) { LogStash::WrappedSynchronousQueue.new(1024, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED) } before do read_client.set_events_metric(metric.namespace([:stats, :events])) @@ -136,7 +136,9 @@ def threaded_read_client .build end - let(:queue) { LogStash::WrappedAckedQueue.new(queue_settings) } + let(:queue) do + LogStash::WrappedAckedQueue.new(queue_settings, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED) + end before do read_client.set_events_metric(metric.namespace([:stats, :events])) diff --git a/logstash-core/spec/logstash/queue_factory_spec.rb b/logstash-core/spec/logstash/queue_factory_spec.rb index 540113412c..56e54dabb8 100644 --- a/logstash-core/spec/logstash/queue_factory_spec.rb +++ b/logstash-core/spec/logstash/queue_factory_spec.rb @@ -30,7 +30,9 @@ LogStash::Setting::NumericSetting.new("queue.checkpoint.acks", 1024), LogStash::Setting::NumericSetting.new("queue.checkpoint.writes", 1024), LogStash::Setting::BooleanSetting.new("queue.checkpoint.retry", false), + LogStash::Setting::StringSetting.new("queue.compression", "none", true, %w(none speed balanced size disabled)), LogStash::Setting::StringSetting.new("pipeline.id", pipeline_id), + LogStash::Setting::StringSetting.new("pipeline.batch.metrics.sampling_mode", "minimal", true, ["disabled", "minimal", "full"]), LogStash::Setting::PositiveIntegerSetting.new("pipeline.batch.size", 125), LogStash::Setting::PositiveIntegerSetting.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum) ] diff --git a/logstash-core/spec/logstash/settings/password_spec.rb b/logstash-core/spec/logstash/settings/password_spec.rb index ca90bc4f9a..d20819c44c 100644 --- a/logstash-core/spec/logstash/settings/password_spec.rb +++ b/logstash-core/spec/logstash/settings/password_spec.rb @@ -18,7 +18,7 @@ require "spec_helper" require "logstash/settings" -describe LogStash::Setting::Password do +describe LogStash::Setting::PasswordSetting do let(:setting_name) { "secure" } subject(:password_setting) { described_class.new(setting_name, nil, true) } @@ -55,7 +55,7 @@ context 'with an invalid non-string value' do let(:setting_value) { 867_5309 } it 'rejects the invalid value' do - expect { password_setting.set(setting_value) }.to raise_error(ArgumentError, "Setting `#{setting_name}` could not coerce non-string value to password") + expect { password_setting.set(setting_value) }.to raise_error(IllegalArgumentException, "Setting `#{setting_name}` could not coerce non-string value to password") expect(password_setting).to_not be_set end end diff --git a/logstash-core/spec/logstash/settings_spec.rb b/logstash-core/spec/logstash/settings_spec.rb index 60f7d8848a..03e3e4ecb2 100644 --- a/logstash-core/spec/logstash/settings_spec.rb +++ b/logstash-core/spec/logstash/settings_spec.rb @@ -299,6 +299,25 @@ expect(LogStash::Setting::ValidatedPassword.new("test.validated.password", password, password_policies)).to_not be_nil end end + + describe "mode WARN" do + let(:password_policies) { super().merge("mode": "WARN") } + + context "when the password does not conform to the policy" do + let(:password) { LogStash::Util::Password.new("NoNumbers!") } + let(:mock_logger) { double("logger") } + + before :each do + allow_any_instance_of(LogStash::Setting::ValidatedPassword).to receive(:logger).and_return(mock_logger) + end + + it "logs a warning on validation failure" do + expect(mock_logger).to receive(:warn).with(a_string_including("Password must contain at least one digit between 0 and 9.")) + + LogStash::Setting::ValidatedPassword.new("test.validated.password", password, password_policies) + end + end + end end context "placeholders in nested logstash.yml" do diff --git a/logstash-core/spec/logstash/util/wrapped_acked_queue_spec.rb b/logstash-core/spec/logstash/util/wrapped_acked_queue_spec.rb index 67093a78ee..19934c7892 100644 --- a/logstash-core/spec/logstash/util/wrapped_acked_queue_spec.rb +++ b/logstash-core/spec/logstash/util/wrapped_acked_queue_spec.rb @@ -64,7 +64,7 @@ .build end - let(:queue) { LogStash::WrappedAckedQueue.new(queue_settings) } + let(:queue) { LogStash::WrappedAckedQueue.new(queue_settings, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED) } after do queue.close diff --git a/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb b/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb index 9e64419580..2f4b771e00 100644 --- a/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb +++ b/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb @@ -19,7 +19,7 @@ require "logstash/instrument/collector" describe LogStash::WrappedSynchronousQueue do - subject {LogStash::WrappedSynchronousQueue.new(5)} + subject {LogStash::WrappedSynchronousQueue.new(5, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED)} describe "queue clients" do context "when requesting a write client" do diff --git a/logstash-core/src/main/java/co/elastic/logstash/api/NamespacedMetric.java b/logstash-core/src/main/java/co/elastic/logstash/api/NamespacedMetric.java index ef2574cad3..2a5fe5b085 100644 --- a/logstash-core/src/main/java/co/elastic/logstash/api/NamespacedMetric.java +++ b/logstash-core/src/main/java/co/elastic/logstash/api/NamespacedMetric.java @@ -51,6 +51,16 @@ public interface NamespacedMetric extends Metric { */ TimerMetric timer(String metric); + /** + * Creates or retrieves a {@link UserMetric} with the provided {@code metric} name, + * using the supplied {@code userMetricFactory}. + * @param metric the name of the metric + * @param userMetricFactory a factory for creating the metric + * @return the resulting metric at the address, whether retrieved or created + * @param the type of metric to create + */ + > USER_METRIC register(String metric, UserMetric.Factory userMetricFactory); + /** * Increment the {@code metric} metric by 1. * diff --git a/logstash-core/src/main/java/co/elastic/logstash/api/UserMetric.java b/logstash-core/src/main/java/co/elastic/logstash/api/UserMetric.java new file mode 100644 index 0000000000..fa8a633faa --- /dev/null +++ b/logstash-core/src/main/java/co/elastic/logstash/api/UserMetric.java @@ -0,0 +1,64 @@ +package co.elastic.logstash.api; + +import java.util.function.Function; + +/** + * A custom metric. + * @param must be jackson-serializable. + * + * NOTE: this interface is experimental and considered internal. + * Its shape may change from one Logstash release to the next. + */ +public interface UserMetric { + VALUE_TYPE getValue(); + + /** + * A {@link UserMetric.Factory} is the primary way to register a custom user-metric + * along-side a null implementation for performance when metrics are disabled. + * + * @param a sub-interface of {@link UserMetric} + */ + interface Factory> { + Class getType(); + USER_METRIC create(String name); + USER_METRIC nullImplementation(); + } + + /** + * A {@link UserMetric.Provider} is an intermediate helper type meant to be statically available by any + * user-provided {@link UserMetric} interface, encapsulating its null implementation and providing + * a way to simply get a {@link UserMetric.Factory} for a given non-null implementation. + * + * @param an interface that extends {@link UserMetric}. + */ + class Provider> { + private final Class type; + private final USER_METRIC nullImplementation; + + public Provider(final Class type, final USER_METRIC nullImplementation) { + assert type.isInterface() : String.format("type must be an interface, got %s", type); + + this.type = type; + this.nullImplementation = nullImplementation; + } + + public Factory getFactory(final Function supplier) { + return new Factory() { + @Override + public USER_METRIC create(final String name) { + return supplier.apply(name); + } + + @Override + public Class getType() { + return type; + } + + @Override + public USER_METRIC nullImplementation() { + return nullImplementation; + } + }; + } + } +} diff --git a/logstash-core/src/main/java/org/logstash/ConvertedMap.java b/logstash-core/src/main/java/org/logstash/ConvertedMap.java index c9f0da606e..c815679d4e 100644 --- a/logstash-core/src/main/java/org/logstash/ConvertedMap.java +++ b/logstash-core/src/main/java/org/logstash/ConvertedMap.java @@ -21,14 +21,26 @@ package org.logstash; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collection; import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; + +import org.jruby.RubyBignum; +import org.jruby.RubyBoolean; +import org.jruby.RubyFixnum; +import org.jruby.RubyFloat; import org.jruby.RubyHash; +import org.jruby.RubyNil; import org.jruby.RubyString; +import org.jruby.RubySymbol; +import org.jruby.ext.bigdecimal.RubyBigDecimal; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.ext.JrubyTimestampExtLibrary; /** *

This class is an internal API and behaves very different from a standard {@link Map}.

@@ -41,7 +53,7 @@ * intern pool to ensure identity match of equivalent strings. * For performance, we keep a global cache of strings that have been interned for use with {@link ConvertedMap}, * and encourage interning through {@link ConvertedMap#internStringForUseAsKey(String)} to avoid - * the performance pentalty of the global string intern pool. + * the performance penalty of the global string intern pool. */ public final class ConvertedMap extends IdentityHashMap { @@ -157,4 +169,112 @@ public Object unconvert() { private static String convertKey(final RubyString key) { return internStringForUseAsKey(key.asJavaString()); } + + public long estimateMemory() { + return values().stream() + .map(this::estimateMemory) + .mapToLong(Long::longValue) + .sum(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private long estimateMemory(Object o) { + if (o instanceof Boolean) { + return Byte.BYTES; + } + if (o instanceof Byte) { + return Byte.BYTES; + } + if (o instanceof Short) { + return Short.BYTES; + } + if (o instanceof Integer) { + return Integer.BYTES; + } + if (o instanceof Long) { + return Long.BYTES; + } + if (o instanceof Float) { + return Float.BYTES; + } + if (o instanceof Double) { + return Double.BYTES; + } + if (o instanceof Character) { + return Character.BYTES; + } + if (o instanceof String) { + return ((String) o).getBytes().length; + } + if (o instanceof RubyString) { + return ((RubyString) o).getBytes().length; + } + + if (o instanceof Collection) { + Collection c = (Collection) o; + long memory = 0L; + for (Object v : c) { + memory += estimateMemory(v); + } + return memory; + } + + if (o instanceof ConvertedMap) { + ConvertedMap c = (ConvertedMap) o; + return c.estimateMemory(); + } + + if (o instanceof Map) { + // this case shouldn't happen because all Map are converted to ConvertedMap + Map m = (Map) o; + long memory = 0L; + for (Map.Entry e : m.entrySet()) { + memory += estimateMemory(e.getKey()); + memory += estimateMemory(e.getValue()); + } + return memory; + } + if (o instanceof JrubyTimestampExtLibrary.RubyTimestamp) { + // wraps an java.time.Instant which is made of long and int + return Long.BYTES + Integer.BYTES; + } + if (o instanceof BigInteger) { + return ((BigInteger) o).toByteArray().length; + } + if (o instanceof BigDecimal) { + // BigInteger has 4 fields, one reference 2 ints (scale and precision) and a long. + return 8 + 2 * Integer.BYTES + Long.BYTES; + } + if (o instanceof RubyBignum) { + RubyBignum rbn = (RubyBignum) o; + return ((RubyFixnum) rbn.size()).getLongValue(); + } + if (o instanceof RubyBigDecimal) { + RubyBigDecimal rbd = (RubyBigDecimal) o; + // wraps a Java BigDecimal so we can return the size of that: + return estimateMemory(rbd.getValue()); + } + if (o instanceof RubyFixnum) { + // like an int value + return Integer.BYTES; + } + if (o instanceof RubyBoolean) { + return Byte.BYTES; + } + if (o instanceof RubyNil) { + return 8 + Integer.BYTES; // object reference, one int + } + if (o instanceof RubySymbol) { + return estimateMemory(((RubySymbol) o).asJavaString()); + } + if (o instanceof RubyFloat) { + return Double.BYTES; + } + + throw new IllegalArgumentException( + "Unsupported type encountered in estimateMemory: " + o.getClass().getName() + + ". Please ensure all objects passed to estimateMemory are of supported types. " + + "Refer to the ConvertedMap.estimateMemory method for the list of supported types." + ); + } } diff --git a/logstash-core/src/main/java/org/logstash/Event.java b/logstash-core/src/main/java/org/logstash/Event.java index e1e9f4db1f..2a9f79ab9b 100644 --- a/logstash-core/src/main/java/org/logstash/Event.java +++ b/logstash-core/src/main/java/org/logstash/Event.java @@ -529,6 +529,7 @@ private void initFailTag(final Object tag) { * and needs to be converted to a list before appending to it. * @param existing Existing Tag * @param tag Tag to add + * */ private void scalarTagFallback(final String existing, final String tag) { final List tags = new ArrayList<>(2); @@ -567,4 +568,16 @@ private static String getCanonicalFieldReference(final FieldReference field) { return path.stream().collect(Collectors.joining("][", "[", "]")); } } + + /** + * @return a byte size estimation of the event, based on the payloads carried by nested data structures, + * without considering the space needed by the JVM to represent the object itself. + * + * */ + public long estimateMemory() { + long total = 0; + total += data.estimateMemory(); + total += metadata.estimateMemory(); + return total; + } } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java new file mode 100644 index 0000000000..9cf159d4ce --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java @@ -0,0 +1,56 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.Metric; +import co.elastic.logstash.api.NamespacedMetric; +import com.github.luben.zstd.Zstd; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Subclasses of {@link AbstractZstdAwareCompressionCodec} are {@link CompressionCodec}s that are capable + * of detecting and decompressing deflate-compressed events. When decoding byte sequences that are NOT + * deflate-compressed, the given bytes are emitted verbatim. + */ +abstract class AbstractZstdAwareCompressionCodec implements CompressionCodec { + // log from the concrete class + protected final Logger logger = LogManager.getLogger(this.getClass()); + + private final IORatioMetric decodeRatioMetric; + private final RelativeSpendMetric decodeTimerMetric; + + public AbstractZstdAwareCompressionCodec(Metric queueMetric) { + final NamespacedMetric decodeNamespace = queueMetric.namespace("compression", "decode"); + decodeRatioMetric = decodeNamespace.namespace("ratio") + .register("lifetime", AtomicIORatioMetric.FACTORY); + decodeTimerMetric = decodeNamespace.namespace("spend") + .register("lifetime", CalculatedRelativeSpendMetric.FACTORY); + } + + @Override + public byte[] decode(byte[] data) { + if (!isZstd(data)) { + decodeRatioMetric.incrementBy(data.length, data.length); + return data; + } + try { + final byte[] decoded = decodeTimerMetric.time(() -> Zstd.decompress(data)); + decodeRatioMetric.incrementBy(data.length, decoded.length); + logger.trace("decoded {} -> {}", data.length, decoded.length); + return decoded; + } catch (Exception e) { + throw new RuntimeException("Exception while decoding", e); + } + } + + private static final byte[] ZSTD_FRAME_MAGIC = { (byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD }; + + static boolean isZstd(byte[] data) { + if (data.length < 4) { return false; } + + for (int i = 0; i < 4; i++) { + if (data[i] != ZSTD_FRAME_MAGIC[i]) { return false; } + } + + return true; + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/AtomicIORatioMetric.java b/logstash-core/src/main/java/org/logstash/ackedqueue/AtomicIORatioMetric.java new file mode 100644 index 0000000000..fc3426a515 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/AtomicIORatioMetric.java @@ -0,0 +1,95 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.UserMetric; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.logstash.instrument.metrics.AbstractMetric; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.util.concurrent.atomic.AtomicReference; + +/** + * It uses {@code long} under the hood, and is capable of handling sustained 1GiB/sec + * for ~272 years before overflowing. + */ +class AtomicIORatioMetric extends AbstractMetric implements IORatioMetric { + + public static UserMetric.Factory FACTORY = IORatioMetric.PROVIDER.getFactory(AtomicIORatioMetric::new); + + private static final MathContext LIMITED_PRECISION = new MathContext(4, RoundingMode.HALF_UP); + private static final ImmutableRatio ZERO = new ImmutableRatio(0L, 0L); + private static final Logger LOGGER = LogManager.getLogger(AtomicIORatioMetric.class); + + private final AtomicReference atomicReference = new AtomicReference<>(ZERO); + private final Logger logger; + + AtomicIORatioMetric(final String name) { + this(name, LOGGER); + } + + AtomicIORatioMetric(final String name, final Logger logger) { + super(name); + this.logger = logger; + } + + @Override + public Value getLifetime() { + return atomicReference.get(); + } + + @Override + public void incrementBy(int bytesIn, int bytesOut) { + if (bytesIn < 0 || bytesOut < 0) { + logger.warn("cannot decrement IORatioMetric {}", this.getName()); + return; + } + this.atomicReference.getAndUpdate((existing) -> doIncrement(existing, bytesIn, bytesOut)); + } + + // test injection + void setTo(long bytesIn, long bytesOut) { + this.atomicReference.set(new ImmutableRatio(bytesIn, bytesOut)); + } + + @Override + public Double getValue() { + final Value snapshot = getLifetime(); + + final BigDecimal bytesIn = BigDecimal.valueOf(snapshot.bytesIn()); + final BigDecimal bytesOut = BigDecimal.valueOf(snapshot.bytesOut()); + + if (bytesIn.signum() == 0) { + return switch(bytesOut.signum()) { + case -1 -> Double.NEGATIVE_INFINITY; + case 1 -> Double.POSITIVE_INFINITY; + default -> Double.NaN; + }; + } + + return bytesOut.divide(bytesIn, LIMITED_PRECISION).doubleValue(); + } + + public void reset() { + this.atomicReference.set(ZERO); + } + + private ImmutableRatio doIncrement(final ImmutableRatio existing, final int bytesIn, final int bytesOut) { + + final long combinedBytesIn = existing.bytesIn() + bytesIn; + final long combinedBytesOut = existing.bytesOut() + bytesOut; + + if (combinedBytesIn < 0 || combinedBytesOut < 0) { + logger.warn("long overflow; precision will be reduced"); + final long reducedBytesIn = Math.addExact(Math.floorDiv(existing.bytesIn(), 2), bytesIn); + final long reducedBytesOut = Math.addExact(Math.floorDiv(existing.bytesOut(), 2), bytesOut); + + return new ImmutableRatio(reducedBytesIn, reducedBytesOut); + } + + return new ImmutableRatio(combinedBytesIn, combinedBytesOut); + } + + public record ImmutableRatio(long bytesIn, long bytesOut) implements Value { } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetric.java b/logstash-core/src/main/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetric.java new file mode 100644 index 0000000000..c70adcba7a --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetric.java @@ -0,0 +1,58 @@ +package org.logstash.ackedqueue; + +import org.logstash.instrument.metrics.AbstractMetric; +import org.logstash.instrument.metrics.UptimeMetric; +import org.logstash.instrument.metrics.timer.TimerMetric; +import org.logstash.instrument.metrics.timer.TimerMetricFactory; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + +class CalculatedRelativeSpendMetric extends AbstractMetric implements RelativeSpendMetric { + private static final MathContext LIMITED_PRECISION = new MathContext(4, RoundingMode.HALF_UP); + + private final TimerMetric spendMetric; + private final UptimeMetric uptimeMetric; + + public static Factory FACTORY = RelativeSpendMetric.PROVIDER.getFactory(CalculatedRelativeSpendMetric::new); + + public CalculatedRelativeSpendMetric(final String name) { + this(name, TimerMetricFactory.getInstance().create(name + ":spend"), new UptimeMetric(name + ":uptime")); + } + + CalculatedRelativeSpendMetric(String name, TimerMetric spendMetric, UptimeMetric uptimeMetric) { + super(name); + this.spendMetric = spendMetric; + this.uptimeMetric = uptimeMetric; + } + + @Override + public T time(ExceptionalSupplier exceptionalSupplier) throws E { + return this.spendMetric.time(exceptionalSupplier); + } + + @Override + public void reportUntrackedMillis(long untrackedMillis) { + this.spendMetric.reportUntrackedMillis(untrackedMillis); + } + + @Override + public Double getValue() { + BigDecimal spend = BigDecimal.valueOf(spendMetric.getValue()); + BigDecimal uptime = BigDecimal.valueOf(uptimeMetric.getValue()); + + if (uptime.signum() == 0) { + switch (spend.signum()) { + case -1: + return Double.NEGATIVE_INFINITY; + case 0: + return 0.0; + case +1: + return Double.POSITIVE_INFINITY; + } + } + + return spend.divide(uptime, LIMITED_PRECISION).doubleValue(); + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java new file mode 100644 index 0000000000..848de0ce9a --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java @@ -0,0 +1,69 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.Metric; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.logstash.ackedqueue.ZstdEnabledCompressionCodec.Goal; +import org.logstash.plugins.NamespacedMetricImpl; + +public interface CompressionCodec { + Logger LOGGER = LogManager.getLogger(CompressionCodec.class); + + byte[] encode(byte[] data); + byte[] decode(byte[] data); + + /** + * The {@link CompressionCodec#NOOP} is a {@link CompressionCodec} that + * does nothing when encoding and decoding. It is only meant to be activated + * as a safety-latch in the event of compression being broken. + */ + CompressionCodec NOOP = new CompressionCodec() { + @Override + public byte[] encode(byte[] data) { + return data; + } + + @Override + public byte[] decode(byte[] data) { + return data; + } + }; + + @FunctionalInterface + interface Factory { + CompressionCodec create(final Metric metric); + default CompressionCodec create() { + return create(NamespacedMetricImpl.getNullMetric()); + } + } + + static CompressionCodec.Factory fromConfigValue(final String configValue, final Logger logger) { + return switch(configValue) { + case "disabled" -> (metric) -> { + logger.warn("compression support has been disabled"); + return CompressionCodec.NOOP; + }; + case "none" -> (metric) -> { + logger.info("compression support is enabled (read-only)"); + return new ZstdAwareCompressionCodec(metric); + }; + case "speed" -> (metric) -> { + logger.info("compression support is enabled (goal: speed)"); + return new ZstdEnabledCompressionCodec(Goal.SPEED, metric); + }; + case "balanced" -> (metric) -> { + logger.info("compression support is enabled (goal: balanced)"); + return new ZstdEnabledCompressionCodec(Goal.BALANCED, metric); + }; + case "size" -> (metric) -> { + logger.info("compression support is enabled (goal: size)"); + return new ZstdEnabledCompressionCodec(Goal.SIZE, metric); + }; + default -> throw new IllegalArgumentException(String.format("Unsupported compression setting `%s`", configValue)); + }; + } + + static CompressionCodec.Factory fromConfigValue(final String configValue) { + return fromConfigValue(configValue, LOGGER); + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/IORatioMetric.java b/logstash-core/src/main/java/org/logstash/ackedqueue/IORatioMetric.java new file mode 100644 index 0000000000..85c8840bdc --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/IORatioMetric.java @@ -0,0 +1,49 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.UserMetric; +import org.logstash.instrument.metrics.MetricType; + +/** + * A {@code IORatioMetric} is a custom metric that tracks the ratio of input to output. + */ +interface IORatioMetric extends UserMetric, org.logstash.instrument.metrics.Metric { + Double getValue(); + + Value getLifetime(); + + void incrementBy(int bytesIn, int bytesOut); + + @Override + default MetricType getType() { + return MetricType.USER; + } + + // NOTE: at 100GiB/sec, this value type has capacity for ~272 years. + interface Value { + long bytesIn(); + + long bytesOut(); + } + + Provider PROVIDER = new Provider<>(IORatioMetric.class, new IORatioMetric() { + @Override + public Double getValue() { + return Double.NaN; + } + + @Override + public Value getLifetime() { + return null; + } + + @Override + public void incrementBy(int bytesIn, int bytesOut) { + // no-op + } + + @Override + public String getName() { + return "NULL"; + } + }); +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java b/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java index 691987793c..42771a1b14 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java @@ -37,6 +37,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import co.elastic.logstash.api.Metric; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.logstash.FileLockFactory; @@ -46,6 +47,7 @@ import org.logstash.ackedqueue.io.MmapPageIOV2; import org.logstash.ackedqueue.io.PageIO; import org.logstash.common.FsUtil; +import org.logstash.plugins.NamespacedMetricImpl; /** * Persistent queue implementation. @@ -83,6 +85,7 @@ public final class Queue implements Closeable { // deserialization private final Class elementClass; private final Method deserializeMethod; + private final CompressionCodec compressionCodec; // thread safety private final ReentrantLock lock = new ReentrantLock(); @@ -95,7 +98,15 @@ public final class Queue implements Closeable { private static final Logger logger = LogManager.getLogger(Queue.class); + private final Metric metric; + + public Queue(Settings settings) { + this(settings, null); + } + + public Queue(Settings settings, Metric metric) { + this.metric = Objects.requireNonNullElseGet(metric, NamespacedMetricImpl::getNullMetric); try { final Path queueDir = Paths.get(settings.getDirPath()); // Files.createDirectories raises a FileAlreadyExistsException @@ -112,6 +123,7 @@ public Queue(Settings settings) { this.maxBytes = settings.getQueueMaxBytes(); this.checkpointIO = new FileCheckpointIO(dirPath, settings.getCheckpointRetry()); this.elementClass = settings.getElementClass(); + this.compressionCodec = settings.getCompressionCodecFactory().create(metric); this.tailPages = new ArrayList<>(); this.unreadTailPages = new ArrayList<>(); this.closed = new AtomicBoolean(true); // not yet opened @@ -414,7 +426,8 @@ public long write(Queueable element) throws IOException { throw new QueueRuntimeException(QueueExceptionMessages.CANNOT_WRITE_TO_CLOSED_QUEUE); } - byte[] data = element.serialize(); + byte[] serializedBytes = element.serialize(); + byte[] data = compressionCodec.encode(serializedBytes); // the write strategy with regard to the isFull() state is to assume there is space for this element // and write it, then after write verify if we just filled the queue and wait on the notFull condition @@ -767,7 +780,8 @@ public CheckpointIO getCheckpointIO() { */ public Queueable deserialize(byte[] bytes) { try { - return (Queueable)this.deserializeMethod.invoke(this.elementClass, bytes); + byte[] decodedBytes = compressionCodec.decode(bytes); + return (Queueable)this.deserializeMethod.invoke(this.elementClass, decodedBytes); } catch (IllegalAccessException|InvocationTargetException e) { throw new QueueRuntimeException("deserialize invocation error", e); } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java index 6a10c2a3e7..de692ee856 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java @@ -24,6 +24,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; + +import co.elastic.logstash.api.NamespacedMetric; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jruby.Ruby; import org.jruby.RubyBasicObject; import org.jruby.RubyClass; @@ -37,6 +41,8 @@ import org.logstash.common.SettingKeyDefinitions; import org.logstash.execution.AbstractWrappedQueueExt; import org.logstash.ext.JrubyWrappedSynchronousQueueExt; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.plugins.NamespacedMetricImpl; import static org.logstash.common.SettingKeyDefinitions.*; @@ -46,6 +52,12 @@ @JRubyClass(name = "QueueFactory") public final class QueueFactoryExt extends RubyBasicObject { + public enum BatchMetricMode { + DISABLED, + MINIMAL, + FULL + } + /** * A static value to indicate Persistent Queue is enabled. */ @@ -63,14 +75,24 @@ public final class QueueFactoryExt extends RubyBasicObject { private static final long serialVersionUID = 1L; + private static final Logger LOGGER = LogManager.getLogger(QueueFactoryExt.class); + public QueueFactoryExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } + @Deprecated @JRubyMethod(meta = true) public static AbstractWrappedQueueExt create(final ThreadContext context, final IRubyObject recv, - final IRubyObject settings) throws IOException { + final IRubyObject settings) throws IOException { + return create(context, settings, null); + } + + public static AbstractWrappedQueueExt create(final ThreadContext context, + final IRubyObject settings, + final AbstractNamespacedMetricExt metric) throws IOException { final String type = getSetting(context, settings, QUEUE_TYPE_CONTEXT_NAME).asJavaString(); + final BatchMetricMode batchMetricMode = decodeBatchMetricMode(context, settings); if (PERSISTED_TYPE.equals(type)) { final Settings queueSettings = extractQueueSettings(settings); final Path queuePath = Paths.get(queueSettings.getDirPath()); @@ -81,29 +103,44 @@ public static AbstractWrappedQueueExt create(final ThreadContext context, final Files.createDirectories(queuePath); } - return JRubyWrappedAckedQueueExt.create(context, queueSettings); + final NamespacedMetric namespacedMetric = getMetric(context, metric); + return JRubyWrappedAckedQueueExt.create(context, queueSettings, namespacedMetric, batchMetricMode); + } else if (MEMORY_TYPE.equals(type)) { - return new JrubyWrappedSynchronousQueueExt( - context.runtime, RubyUtil.WRAPPED_SYNCHRONOUS_QUEUE_CLASS - ).initialize( - context, context.runtime.newFixnum( - getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) - .convertToInteger().getIntValue() - * getSetting(context, settings, SettingKeyDefinitions.PIPELINE_WORKERS) - .convertToInteger().getIntValue() - ) - ); + final int batchSize = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) + .convertToInteger().getIntValue(); + final int workers = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_WORKERS) + .convertToInteger().getIntValue(); + int queueSize = batchSize * workers; + return JrubyWrappedSynchronousQueueExt.create(context, queueSize, batchMetricMode); } else { throw context.runtime.newRaiseException( - RubyUtil.CONFIGURATION_ERROR_CLASS, - String.format( - "Invalid setting `%s` for `queue.type`, supported types are: 'memory' or 'persisted'", - type - ) + RubyUtil.CONFIGURATION_ERROR_CLASS, + String.format( + "Invalid setting `%s` for `queue.type`, supported types are: 'memory' or 'persisted'", + type + ) ); } } + private static BatchMetricMode decodeBatchMetricMode(ThreadContext context, IRubyObject settings) { + final String batchMetricModeStr = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_METRICS) + .asJavaString(); + + if (batchMetricModeStr == null || batchMetricModeStr.isEmpty()) { + return BatchMetricMode.DISABLED; + } + return BatchMetricMode.valueOf(batchMetricModeStr.toUpperCase()); + } + + private static NamespacedMetric getMetric(final ThreadContext context, final AbstractNamespacedMetricExt metric) { + if ( metric == null ) { + return NamespacedMetricImpl.getNullMetric(); + } + return new NamespacedMetricImpl(context, metric); + } + private static IRubyObject getSetting(final ThreadContext context, final IRubyObject settings, final String name) { return settings.callMethod(context, "get_value", context.runtime.newString(name)); @@ -115,6 +152,7 @@ private static Settings extractQueueSettings(final IRubyObject settings) { getSetting(context, settings, PATH_QUEUE).asJavaString(), getSetting(context, settings, PIPELINE_ID).asJavaString() ); + return SettingsImpl.fileSettingsBuilder(queuePath.toString()) .elementClass(Event.class) .capacity(getSetting(context, settings, QUEUE_PAGE_CAPACITY).toJava(Integer.class)) @@ -123,6 +161,13 @@ private static Settings extractQueueSettings(final IRubyObject settings) { .checkpointMaxAcks(getSetting(context, settings, QUEUE_CHECKPOINT_ACKS).toJava(Integer.class)) .checkpointRetry(getSetting(context, settings, QUEUE_CHECKPOINT_RETRY).isTrue()) .queueMaxBytes(getSetting(context, settings, QUEUE_MAX_BYTES).toJava(Integer.class)) + .compressionCodecFactory(extractConfiguredCodec(settings)) .build(); } + + private static CompressionCodec.Factory extractConfiguredCodec(final IRubyObject settings) { + final ThreadContext context = settings.getRuntime().getCurrentContext(); + final String compressionSetting = getSetting(context, settings, QUEUE_COMPRESSION).asJavaString(); + return CompressionCodec.fromConfigValue(compressionSetting, LOGGER); + } } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/RelativeSpendMetric.java b/logstash-core/src/main/java/org/logstash/ackedqueue/RelativeSpendMetric.java new file mode 100644 index 0000000000..615f5612c6 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/RelativeSpendMetric.java @@ -0,0 +1,35 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.TimerMetric; +import co.elastic.logstash.api.UserMetric; +import org.logstash.instrument.metrics.MetricType; +import org.logstash.instrument.metrics.timer.NullTimerMetric; + +interface RelativeSpendMetric extends UserMetric, org.logstash.instrument.metrics.Metric, TimerMetric { + + default MetricType getType() { + return MetricType.USER; + } + + Provider PROVIDER = new Provider<>(RelativeSpendMetric.class, new RelativeSpendMetric() { + @Override + public T time(ExceptionalSupplier exceptionalSupplier) throws E { + return NullTimerMetric.getInstance().time(exceptionalSupplier); + } + + @Override + public void reportUntrackedMillis(long untrackedMillis) { + // no-op + } + + @Override + public Double getValue() { + return 0.0; + } + + @Override + public String getName() { + return "NULL"; + } + }); +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java b/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java index 1623738659..2e4646281f 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java @@ -44,6 +44,8 @@ public interface Settings { boolean getCheckpointRetry(); + CompressionCodec.Factory getCompressionCodecFactory(); + /** * Validate and return the settings, or throw descriptive {@link QueueRuntimeException} * @param settings the settings to validate @@ -89,7 +91,8 @@ interface Builder { Builder checkpointRetry(boolean checkpointRetry); - Settings build(); + Builder compressionCodecFactory(CompressionCodec.Factory compressionCodecFactory); + Settings build(); } } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java b/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java index bc191f44a3..c5a95c3b67 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java @@ -31,6 +31,7 @@ public class SettingsImpl implements Settings { private final int checkpointMaxAcks; private final int checkpointMaxWrites; private final boolean checkpointRetry; + private final CompressionCodec.Factory compressionCodec; public static Builder builder(final Settings settings) { return new BuilderImpl(settings); @@ -49,6 +50,7 @@ private SettingsImpl(final BuilderImpl builder) { this.checkpointMaxAcks = builder.checkpointMaxAcks; this.checkpointMaxWrites = builder.checkpointMaxWrites; this.checkpointRetry = builder.checkpointRetry; + this.compressionCodec = builder.compressionCodecFactory; } @Override @@ -91,6 +93,11 @@ public boolean getCheckpointRetry() { return this.checkpointRetry; } + @Override + public CompressionCodec.Factory getCompressionCodecFactory() { + return this.compressionCodec; + } + /** * Default implementation for Setting's Builder * */ @@ -140,6 +147,8 @@ private static final class BuilderImpl implements Builder { private boolean checkpointRetry; + private CompressionCodec.Factory compressionCodecFactory; + private BuilderImpl(final String dirForFiles) { this.dirForFiles = dirForFiles; this.elementClass = null; @@ -148,6 +157,7 @@ private BuilderImpl(final String dirForFiles) { this.maxUnread = DEFAULT_MAX_UNREAD; this.checkpointMaxAcks = DEFAULT_CHECKPOINT_MAX_ACKS; this.checkpointMaxWrites = DEFAULT_CHECKPOINT_MAX_WRITES; + this.compressionCodecFactory = (metric) -> CompressionCodec.NOOP; this.checkpointRetry = false; } @@ -160,6 +170,7 @@ private BuilderImpl(final Settings settings) { this.checkpointMaxAcks = settings.getCheckpointMaxAcks(); this.checkpointMaxWrites = settings.getCheckpointMaxWrites(); this.checkpointRetry = settings.getCheckpointRetry(); + this.compressionCodecFactory = settings.getCompressionCodecFactory(); } @Override @@ -204,6 +215,12 @@ public Builder checkpointRetry(final boolean checkpointRetry) { return this; } + @Override + public Builder compressionCodecFactory(CompressionCodec.Factory compressionCodec) { + this.compressionCodecFactory = compressionCodec; + return this; + } + @Override public Settings build() { return Settings.ensureValid(new SettingsImpl(this)); diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java new file mode 100644 index 0000000000..c4797e4144 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java @@ -0,0 +1,19 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.Metric; + +/** + * A {@link ZstdAwareCompressionCodec} is an {@link CompressionCodec} that can decode deflate-compressed + * bytes, but performs no compression when encoding. + */ +class ZstdAwareCompressionCodec extends AbstractZstdAwareCompressionCodec { + + public ZstdAwareCompressionCodec(Metric queueMetric) { + super(queueMetric); + } + + @Override + public byte[] encode(byte[] data) { + return data; + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java new file mode 100644 index 0000000000..510bb0cca6 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java @@ -0,0 +1,57 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.Metric; +import co.elastic.logstash.api.NamespacedMetric; +import com.github.luben.zstd.Zstd; + +import java.util.Locale; + +/** + * A {@link ZstdEnabledCompressionCodec} is a {@link CompressionCodec} that can decode deflate-compressed + * bytes and performs deflate compression when encoding. + */ +class ZstdEnabledCompressionCodec extends AbstractZstdAwareCompressionCodec implements CompressionCodec { + public enum Goal { + FASTEST(-7), + SPEED(-1), + BALANCED(3), + HIGH(14), + SIZE(22), + ; + + private int internalLevel; + + Goal(final int internalLevel) { + this.internalLevel = internalLevel; + } + } + + private final int internalLevel; + + private final IORatioMetric encodeRatioMetric; + private final RelativeSpendMetric encodeTimerMetric; + + ZstdEnabledCompressionCodec(final Goal internalLevel, final Metric queueMetric) { + super(queueMetric); + this.internalLevel = internalLevel.internalLevel; + + final NamespacedMetric encodeNamespace = queueMetric.namespace("compression", "encode"); + encodeNamespace.gauge("goal", internalLevel.name().toLowerCase(Locale.ROOT)); + encodeRatioMetric = encodeNamespace.namespace("ratio") + .register("lifetime", AtomicIORatioMetric.FACTORY); + encodeTimerMetric = encodeNamespace.namespace("spend") + .register("lifetime", CalculatedRelativeSpendMetric.FACTORY); + } + + @Override + public byte[] encode(byte[] data) { + try { + final byte[] encoded = encodeTimerMetric.time(() -> Zstd.compress(data, internalLevel)); + encodeRatioMetric.incrementBy(data.length, encoded.length); + logger.trace("encoded {} -> {}", data.length, encoded.length); + return encoded; + } catch (Exception e) { + throw new RuntimeException("Exception while encoding", e); + } + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java index 35cb765dbc..d98d49a336 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Objects; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyBoolean; import org.jruby.RubyClass; @@ -42,9 +43,11 @@ import org.logstash.ackedqueue.QueueExceptionMessages; import org.logstash.ackedqueue.Settings; import org.logstash.ackedqueue.SettingsImpl; +import org.logstash.plugins.NamespacedMetricImpl; + /** - * JRuby extension to wrap a persistent queue istance. + * JRuby extension to wrap a persistent queue instance. */ @JRubyClass(name = "AckedQueue") public final class JRubyAckedQueueExt extends RubyObject { @@ -62,9 +65,14 @@ public Queue getQueue() { return this.queue; } + @Deprecated public static JRubyAckedQueueExt create(final Settings settings) { + return create(settings, NamespacedMetricImpl.getNullMetric()); + } + + public static JRubyAckedQueueExt create(final Settings settings, final Metric metric) { JRubyAckedQueueExt queueExt = new JRubyAckedQueueExt(RubyUtil.RUBY, RubyUtil.ACKED_QUEUE_CLASS); - queueExt.queue = new Queue(settings); + queueExt.queue = new Queue(settings, metric); return queueExt; } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java index d2d374b56f..ca58436bd4 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java @@ -21,7 +21,9 @@ package org.logstash.ackedqueue.ext; import java.io.IOException; +import java.util.Objects; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyBoolean; import org.jruby.RubyClass; @@ -32,12 +34,15 @@ import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; import org.logstash.ackedqueue.Settings; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.execution.AbstractWrappedQueueExt; import org.logstash.execution.QueueReadClientBase; import org.logstash.ext.JRubyAbstractQueueWriteClientExt; import org.logstash.ext.JrubyAckedReadClientExt; import org.logstash.ext.JrubyAckedWriteClientExt; import org.logstash.ext.JrubyEventExtLibrary; +import org.logstash.instrument.metrics.AbstractMetricExt; +import org.logstash.plugins.NamespacedMetricImpl; /** * JRuby extension @@ -48,9 +53,11 @@ public final class JRubyWrappedAckedQueueExt extends AbstractWrappedQueueExt { private static final long serialVersionUID = 1L; private JRubyAckedQueueExt queue; + private QueueFactoryExt.BatchMetricMode batchMetricMode; - @JRubyMethod(required=1) - public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject settings) throws IOException { + @JRubyMethod(required=2, optional=1) + public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject[] args) throws IOException { + final IRubyObject settings = args[0]; if (!JavaUtil.isJavaObject(settings)) { // We should never get here, but previously had an initialize method // that took 7 technically-optional ordered parameters. @@ -60,19 +67,45 @@ public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject s settings.getClass().getName(), settings)); } - this.queue = JRubyAckedQueueExt.create(JavaUtil.unwrapJavaObject(settings)); + + final IRubyObject batchMetricMode = args[1]; + Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); + if (!JavaUtil.isJavaObject(batchMetricMode)) { + throw new IllegalArgumentException( + String.format( + "Failed to instantiate JRubyWrappedAckedQueueExt with <%s:%s>", + batchMetricMode.getClass().getName(), + batchMetricMode)); + } + + final Metric metric = getApiMetric(args.length > 2 ? args[2] : null); + + Settings javaSettings = JavaUtil.unwrapJavaObject(settings); + this.queue = JRubyAckedQueueExt.create(javaSettings, metric); + + this.batchMetricMode = JavaUtil.unwrapJavaObject(batchMetricMode); this.queue.open(); return this; } - public static JRubyWrappedAckedQueueExt create(ThreadContext context, Settings settings) throws IOException { - return new JRubyWrappedAckedQueueExt(context.runtime, RubyUtil.WRAPPED_ACKED_QUEUE_CLASS, settings); + public static JRubyWrappedAckedQueueExt create(ThreadContext context, Settings settings, Metric metric, QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { + return new JRubyWrappedAckedQueueExt(context.runtime, RubyUtil.WRAPPED_ACKED_QUEUE_CLASS, settings, metric, batchMetricMode); } - public JRubyWrappedAckedQueueExt(Ruby runtime, RubyClass metaClass, Settings settings) throws IOException { + @Deprecated + public JRubyWrappedAckedQueueExt(Ruby runtime, RubyClass metaClass, Settings settings, QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { + this(runtime, metaClass, settings, NamespacedMetricImpl.getNullMetric(), batchMetricMode); + } + + public JRubyWrappedAckedQueueExt(final Ruby runtime, + final RubyClass metaClass, + final Settings settings, + final Metric metric, + final QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { super(runtime, metaClass); - this.queue = JRubyAckedQueueExt.create(settings); + this.batchMetricMode = Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); + this.queue = JRubyAckedQueueExt.create(settings, metric); this.queue.open(); } @@ -80,6 +113,19 @@ public JRubyWrappedAckedQueueExt(final Ruby runtime, final RubyClass metaClass) super(runtime, metaClass); } + private static Metric getApiMetric(IRubyObject metric) { + if (Objects.isNull(metric) || metric.isNil()) { + return NamespacedMetricImpl.getNullMetric(); + } + if (metric instanceof AbstractMetricExt rubyExtensionMetric) { + return rubyExtensionMetric.asApiMetric(); + } + if (Metric.class.isAssignableFrom(metric.getJavaClass())) { + return metric.toJava(Metric.class); + } + throw new IllegalArgumentException(String.format("Object <%s> could not be converted to a metric", metric.inspect())); + } + @JRubyMethod(name = "queue") public JRubyAckedQueueExt rubyGetQueue() { return queue; @@ -111,7 +157,7 @@ protected JRubyAbstractQueueWriteClientExt getWriteClient(final ThreadContext co @Override protected QueueReadClientBase getReadClient() { - return JrubyAckedReadClientExt.create(queue); + return JrubyAckedReadClientExt.create(queue, batchMetricMode); } @Override diff --git a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java index 7c5c47e316..b6a426db34 100644 --- a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java +++ b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java @@ -28,6 +28,8 @@ public class SettingKeyDefinitions { public static final String PIPELINE_WORKERS = "pipeline.workers"; + public static final String PIPELINE_BATCH_METRICS = "pipeline.batch.metrics.sampling_mode"; + public static final String PIPELINE_BATCH_SIZE = "pipeline.batch.size"; public static final String PATH_QUEUE = "path.queue"; @@ -43,4 +45,6 @@ public class SettingKeyDefinitions { public static final String QUEUE_CHECKPOINT_RETRY = "queue.checkpoint.retry"; public static final String QUEUE_MAX_BYTES = "queue.max_bytes"; + + public static final String QUEUE_COMPRESSION = "queue.compression"; } \ No newline at end of file diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index ec26b9b073..b90b77de54 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -92,6 +92,7 @@ import org.logstash.instrument.metrics.MetricType; import org.logstash.instrument.metrics.NullMetricExt; import org.logstash.instrument.metrics.UpScaledMetric; +import org.logstash.instrument.metrics.gauge.TextGauge; import org.logstash.instrument.metrics.timer.TimerMetric; import org.logstash.instrument.metrics.UptimeMetric; import org.logstash.instrument.metrics.counter.LongCounter; @@ -299,8 +300,9 @@ private AbstractPipelineExt initialize(final ThreadContext context, */ @JRubyMethod(name = "open_queue") public final IRubyObject openQueue(final ThreadContext context) { + final AbstractNamespacedMetricExt queueNamespace = metric.namespace(context, pipelineNamespacedPath(QUEUE_KEY)); try { - queue = QueueFactoryExt.create(context, null, settings); + queue = QueueFactoryExt.create(context, settings, queueNamespace); } catch (final Exception ex) { LOGGER.error("Logstash failed to create queue.", ex); throw new IllegalStateException(ex); @@ -317,8 +319,7 @@ public final IRubyObject openQueue(final ThreadContext context) { new IRubyObject[]{ STATS_KEY, PIPELINES_KEY, - pipelineId.convertToString().intern(), - EVENTS_KEY + pipelineId.convertToString().intern() } ) ) @@ -585,11 +586,39 @@ public final IRubyObject initializeFlowMetrics(final ThreadContext context) { this.scopedFlowMetrics.register(ScopedFlowMetrics.Scope.WORKER, utilizationFlow); storeMetric(context, flowNamespace, utilizationFlow); + // Batch average byte size and count metrics + if (isBatchMetricsEnabled(context)) { + initializeBatchMetrics(context); + } + initializePqFlowMetrics(context, flowNamespace, uptimeMetric); initializePluginFlowMetrics(context, uptimeMetric); return context.nil; } + private void initializeBatchMetrics(ThreadContext context) { + final RubySymbol[] batchNamespace = buildNamespace(BATCH_KEY, BATCH_EVENT_COUNT_KEY); + final LongCounter batchEventsInCounter = initOrGetCounterMetric(context, buildNamespace(BATCH_KEY), BATCH_TOTAL_EVENTS); + final LongCounter batchCounter = initOrGetCounterMetric(context, buildNamespace(BATCH_KEY), BATCH_COUNT); + final FlowMetric documentsPerBatch = createFlowMetric(BATCH_AVERAGE_KEY, batchEventsInCounter, batchCounter); + this.scopedFlowMetrics.register(ScopedFlowMetrics.Scope.WORKER, documentsPerBatch); + storeMetric(context, batchNamespace, documentsPerBatch); + + final RubySymbol[] batchSizeNamespace = buildNamespace(BATCH_KEY, BATCH_BYTE_SIZE_KEY); + final LongCounter totalBytes = initOrGetCounterMetric(context, buildNamespace(BATCH_KEY), BATCH_TOTAL_BYTES); + final FlowMetric byteSizePerBatch = createFlowMetric(BATCH_AVERAGE_KEY, totalBytes, batchCounter); + this.scopedFlowMetrics.register(ScopedFlowMetrics.Scope.WORKER, byteSizePerBatch); + storeMetric(context, batchSizeNamespace, byteSizePerBatch); + } + + private boolean isBatchMetricsEnabled(ThreadContext context) { + IRubyObject pipelineBatchMetricsSetting = getSetting(context, "pipeline.batch.metrics.sampling_mode"); + return !pipelineBatchMetricsSetting.isNil() && + QueueFactoryExt.BatchMetricMode.valueOf( + pipelineBatchMetricsSetting.asJavaString().toUpperCase() + ) != QueueFactoryExt.BatchMetricMode.DISABLED; + } + @JRubyMethod(name = "collect_flow_metrics") public final IRubyObject collectFlowMetrics(final ThreadContext context) { this.scopedFlowMetrics.captureAll(); diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java index 535cd838a0..aba1377ca5 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java @@ -32,15 +32,19 @@ import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; import org.logstash.instrument.metrics.MetricKeys; import org.logstash.instrument.metrics.timer.TimerMetric; import org.logstash.instrument.metrics.counter.LongCounter; import java.io.IOException; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import static org.logstash.instrument.metrics.MetricKeys.EVENTS_KEY; + /** * Common code shared by Persistent and In-Memory queues clients implementation * */ @@ -60,9 +64,17 @@ public abstract class QueueReadClientBase extends RubyObject implements QueueRea private transient LongCounter pipelineMetricOut; private transient LongCounter pipelineMetricFiltered; private transient TimerMetric pipelineMetricTime; + private final transient QueueReadClientBatchMetrics batchMetrics; protected QueueReadClientBase(final Ruby runtime, final RubyClass metaClass) { + this(runtime, metaClass, QueueFactoryExt.BatchMetricMode.DISABLED); + } + + protected QueueReadClientBase(final Ruby runtime, final RubyClass metaClass, + final QueueFactoryExt.BatchMetricMode batchMetricMode) { super(runtime, metaClass); + Objects.requireNonNull(batchMetricMode, "batchMetricMode must not be null"); + this.batchMetrics = new QueueReadClientBatchMetrics(batchMetricMode); } @JRubyMethod(name = "inflight_batches") @@ -86,10 +98,13 @@ public IRubyObject setEventsMetric(final IRubyObject metric) { @JRubyMethod(name = "set_pipeline_metric") public IRubyObject setPipelineMetric(final IRubyObject metric) { final AbstractNamespacedMetricExt namespacedMetric = (AbstractNamespacedMetricExt) metric; + ThreadContext context = metric.getRuntime().getCurrentContext(); + AbstractNamespacedMetricExt eventsNamespace = namespacedMetric.namespace(context, EVENTS_KEY); synchronized(namespacedMetric.getMetric()) { - pipelineMetricOut = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.OUT_KEY); - pipelineMetricFiltered = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.FILTERED_KEY); - pipelineMetricTime = TimerMetric.fromRubyBase(namespacedMetric, MetricKeys.DURATION_IN_MILLIS_KEY); + pipelineMetricOut = LongCounter.fromRubyBase(eventsNamespace, MetricKeys.OUT_KEY); + pipelineMetricFiltered = LongCounter.fromRubyBase(eventsNamespace, MetricKeys.FILTERED_KEY); + pipelineMetricTime = TimerMetric.fromRubyBase(eventsNamespace, MetricKeys.DURATION_IN_MILLIS_KEY); + batchMetrics.setupMetrics(namespacedMetric); } return this; } @@ -193,6 +208,7 @@ public void startMetrics(QueueBatch batch) { // JTODO getId has been deprecated in JDK 19, when JDK 21 is the target version use threadId() instead long threadId = Thread.currentThread().getId(); inflightBatches.put(threadId, batch); + batchMetrics.updateBatchMetrics(batch); } @Override diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java new file mode 100644 index 0000000000..a91cdf5ded --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java @@ -0,0 +1,82 @@ +package org.logstash.execution; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jruby.runtime.ThreadContext; +import org.logstash.ackedqueue.QueueFactoryExt; +import org.logstash.ext.JrubyEventExtLibrary; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.counter.LongCounter; +import org.logstash.instrument.metrics.gauge.LazyDelegatingGauge; + +import java.security.SecureRandom; +import java.util.Arrays; + +import static org.logstash.instrument.metrics.MetricKeys.*; + +class QueueReadClientBatchMetrics { + + private static final Logger LOG = LogManager.getLogger(QueueReadClientBatchMetrics.class); + + private final QueueFactoryExt.BatchMetricMode batchMetricMode; + + private LongCounter pipelineMetricBatchCount; + private LongCounter pipelineMetricBatchByteSize; + private LongCounter pipelineMetricBatchTotalEvents; + private final SecureRandom random = new SecureRandom(); + private LazyDelegatingGauge currentBatchDimensions; + + public QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode batchMetricMode) { + this.batchMetricMode = batchMetricMode; + } + + public void setupMetrics(AbstractNamespacedMetricExt namespacedMetric) { + LOG.debug("setupMetrics called with mode: {}", batchMetricMode); + ThreadContext context = namespacedMetric.getRuntime().getCurrentContext(); + AbstractNamespacedMetricExt batchNamespace = namespacedMetric.namespace(context, BATCH_KEY); + if (batchMetricMode != QueueFactoryExt.BatchMetricMode.DISABLED) { + pipelineMetricBatchCount = LongCounter.fromRubyBase(batchNamespace, BATCH_COUNT); + pipelineMetricBatchTotalEvents = LongCounter.fromRubyBase(batchNamespace, BATCH_TOTAL_EVENTS); + pipelineMetricBatchByteSize = LongCounter.fromRubyBase(batchNamespace, BATCH_TOTAL_BYTES); + currentBatchDimensions = LazyDelegatingGauge.fromRubyBase(batchNamespace, BATCH_CURRENT_KEY); + } + } + + public void updateBatchMetrics(QueueBatch batch) { + if (batchMetricMode == QueueFactoryExt.BatchMetricMode.DISABLED) { + return; + } + + if (batch.events().isEmpty()) { + // don't update averages for empty batches, but set current back to zero + currentBatchDimensions.set(Arrays.asList(0L, 0L)); + return; + } + + boolean updateMetric = true; + if (batchMetricMode == QueueFactoryExt.BatchMetricMode.MINIMAL) { + // 1% chance to update metric + updateMetric = random.nextInt(100) < 2; + } + + if (updateMetric) { + updateBatchSizeMetric(batch); + } + } + + private void updateBatchSizeMetric(QueueBatch batch) { + try { + // if an error occurs in estimating the size of the batch, no counter has to be updated + long totalByteSize = 0L; + for (JrubyEventExtLibrary.RubyEvent rubyEvent : batch.events()) { + totalByteSize += rubyEvent.getEvent().estimateMemory(); + } + pipelineMetricBatchCount.increment(); + pipelineMetricBatchTotalEvents.increment(batch.filteredSize()); + pipelineMetricBatchByteSize.increment(totalByteSize); + currentBatchDimensions.set(Arrays.asList(batch.filteredSize(), totalByteSize)); + } catch (IllegalArgumentException e) { + LOG.error("Failed to calculate batch byte size for metrics", e); + } + } +} diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyAckedReadClientExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyAckedReadClientExt.java index 273fa6cb2b..3f4f1b62a4 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyAckedReadClientExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyAckedReadClientExt.java @@ -28,6 +28,7 @@ import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; import org.logstash.ackedqueue.AckedReadBatch; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.ackedqueue.ext.JRubyAckedQueueExt; import org.logstash.execution.QueueBatch; import org.logstash.execution.QueueReadClient; @@ -49,12 +50,12 @@ public final class JrubyAckedReadClientExt extends QueueReadClientBase implement public static JrubyAckedReadClientExt create(final ThreadContext context, final IRubyObject recv, final IRubyObject queue) { return new JrubyAckedReadClientExt( - context.runtime, RubyUtil.ACKED_READ_CLIENT_CLASS, queue + context.runtime, RubyUtil.ACKED_READ_CLIENT_CLASS, queue, QueueFactoryExt.BatchMetricMode.DISABLED ); } - public static JrubyAckedReadClientExt create(IRubyObject queue) { - return new JrubyAckedReadClientExt(RubyUtil.RUBY, RubyUtil.ACKED_READ_CLIENT_CLASS, queue); + public static JrubyAckedReadClientExt create(IRubyObject queue, QueueFactoryExt.BatchMetricMode batchMetricMode) { + return new JrubyAckedReadClientExt(RubyUtil.RUBY, RubyUtil.ACKED_READ_CLIENT_CLASS, queue, batchMetricMode); } public JrubyAckedReadClientExt(final Ruby runtime, final RubyClass metaClass) { @@ -62,8 +63,8 @@ public JrubyAckedReadClientExt(final Ruby runtime, final RubyClass metaClass) { } private JrubyAckedReadClientExt(final Ruby runtime, final RubyClass metaClass, - final IRubyObject queue) { - super(runtime, metaClass); + final IRubyObject queue, final QueueFactoryExt.BatchMetricMode batchMetricMode) { + super(runtime, metaClass, batchMetricMode); this.queue = (JRubyAckedQueueExt)queue; } diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java index 0a93a347c4..8485d0864d 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java @@ -26,6 +26,7 @@ import org.jruby.RubyClass; import org.jruby.anno.JRubyClass; import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.common.LsQueueUtils; import org.logstash.execution.MemoryReadBatch; import org.logstash.execution.QueueBatch; @@ -47,8 +48,9 @@ public JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass) { @SuppressWarnings("rawtypes") private JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass, - BlockingQueue queue, int batchSize, int waitForMillis) { - super(runtime, metaClass); + BlockingQueue queue, int batchSize, int waitForMillis, + QueueFactoryExt.BatchMetricMode batchMetricMode) { + super(runtime, metaClass, batchMetricMode); this.queue = queue; this.batchSize = batchSize; this.waitForNanos = TimeUnit.NANOSECONDS.convert(waitForMillis, TimeUnit.MILLISECONDS); @@ -58,8 +60,15 @@ private JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass, @SuppressWarnings("rawtypes") public static JrubyMemoryReadClientExt create(BlockingQueue queue, int batchSize, int waitForMillis) { + return create(queue, batchSize, waitForMillis, QueueFactoryExt.BatchMetricMode.DISABLED); + } + + @SuppressWarnings("rawtypes") + public static JrubyMemoryReadClientExt create(BlockingQueue queue, int batchSize, + int waitForMillis, + QueueFactoryExt.BatchMetricMode batchMetricMode) { return new JrubyMemoryReadClientExt(RubyUtil.RUBY, - RubyUtil.MEMORY_READ_CLIENT_CLASS, queue, batchSize, waitForMillis); + RubyUtil.MEMORY_READ_CLIENT_CLASS, queue, batchSize, waitForMillis, batchMetricMode); } @Override diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java index dbbfb97de5..ebebae00cb 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java @@ -20,6 +20,7 @@ package org.logstash.ext; +import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -28,8 +29,11 @@ import org.jruby.RubyNumeric; import org.jruby.anno.JRubyClass; import org.jruby.anno.JRubyMethod; +import org.jruby.javasupport.JavaUtil; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.execution.AbstractWrappedQueueExt; import org.logstash.execution.QueueReadClientBase; @@ -42,20 +46,44 @@ public final class JrubyWrappedSynchronousQueueExt extends AbstractWrappedQueueE private static final long serialVersionUID = 1L; private transient BlockingQueue queue; + private QueueFactoryExt.BatchMetricMode batchMetricMode; public JrubyWrappedSynchronousQueueExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } + private JrubyWrappedSynchronousQueueExt(final Ruby runtime, final RubyClass metaClass, + int size, QueueFactoryExt.BatchMetricMode batchMetricMode) { + super(runtime, metaClass); + this.queue = new ArrayBlockingQueue<>(size); + this.batchMetricMode = Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); + } + @JRubyMethod @SuppressWarnings("unchecked") public JrubyWrappedSynchronousQueueExt initialize(final ThreadContext context, - IRubyObject size) { + IRubyObject size, + IRubyObject batchMetricMode) { + if (!JavaUtil.isJavaObject(batchMetricMode)) { + throw new IllegalArgumentException( + String.format( + "Failed to instantiate JrubyWrappedSynchronousQueueExt with <%s:%s>", + batchMetricMode.getClass().getName(), + batchMetricMode)); + } + int typedSize = ((RubyNumeric)size).getIntValue(); this.queue = new ArrayBlockingQueue<>(typedSize); + Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); + this.batchMetricMode = JavaUtil.unwrapJavaObject(batchMetricMode); return this; } + public static JrubyWrappedSynchronousQueueExt create(final ThreadContext context, int size, + QueueFactoryExt.BatchMetricMode batchMetricMode) { + return new JrubyWrappedSynchronousQueueExt(context.runtime, RubyUtil.WRAPPED_SYNCHRONOUS_QUEUE_CLASS, size, batchMetricMode); + } + @Override protected JRubyAbstractQueueWriteClientExt getWriteClient(final ThreadContext context) { return JrubyMemoryWriteClientExt.create(queue); @@ -65,7 +93,7 @@ protected JRubyAbstractQueueWriteClientExt getWriteClient(final ThreadContext co protected QueueReadClientBase getReadClient() { // batch size and timeout are currently hard-coded to 125 and 50ms as values observed // to be reasonable tradeoffs between latency and throughput per PR #8707 - return JrubyMemoryReadClientExt.create(queue, 125, 50); + return JrubyMemoryReadClientExt.create(queue, 125, 50, batchMetricMode); } @Override diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractMetricExt.java index a0298faf18..7b1ada37c1 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractMetricExt.java @@ -52,6 +52,8 @@ public final IRubyObject collector(final ThreadContext context) { return getCollector(context); } + public abstract co.elastic.logstash.api.Metric asApiMetric(); + protected abstract AbstractNamespacedMetricExt createNamespaced( ThreadContext context, IRubyObject name ); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java index 1a24b3cc03..4dfc8b4855 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java @@ -20,6 +20,7 @@ package org.logstash.instrument.metrics; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyClass; @@ -28,6 +29,7 @@ import org.jruby.runtime.Block; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.plugins.NamespacedMetricImpl; @JRubyClass(name = "AbstractNamespacedMetric") public abstract class AbstractNamespacedMetricExt extends AbstractMetricExt { @@ -53,6 +55,11 @@ public IRubyObject timer(final ThreadContext context, final IRubyObject key) { return getTimer(context, key); } + @JRubyMethod + public IRubyObject register(final ThreadContext context, final IRubyObject key, final Block metricSupplier) { + return doRegister(context, key, metricSupplier); + } + @JRubyMethod(required = 1, optional = 1) public IRubyObject increment(final ThreadContext context, final IRubyObject[] args) { return doIncrement(context, args); @@ -104,5 +111,12 @@ protected abstract IRubyObject doReportTime(ThreadContext context, protected abstract IRubyObject doDecrement(ThreadContext context, IRubyObject[] args); + @Override + public Metric asApiMetric() { + return new NamespacedMetricImpl(getRuntime().getCurrentContext(), this); + } + + protected abstract IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier); + public abstract AbstractMetricExt getMetric(); } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java index 758d9309ea..6f3553f72a 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java @@ -20,6 +20,7 @@ package org.logstash.instrument.metrics; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyClass; import org.jruby.anno.JRubyClass; @@ -27,6 +28,7 @@ import org.jruby.runtime.Block; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.plugins.RootMetricImpl; @JRubyClass(name = "AbstractSimpleMetric") public abstract class AbstractSimpleMetricExt extends AbstractMetricExt { @@ -37,6 +39,11 @@ public abstract class AbstractSimpleMetricExt extends AbstractMetricExt { super(runtime, metaClass); } + @Override + public Metric asApiMetric() { + return new RootMetricImpl(getRuntime().getCurrentContext(), this); + } + @JRubyMethod(required = 2, optional = 1) public IRubyObject increment(final ThreadContext context, final IRubyObject[] args) { return doIncrement(context, args); @@ -74,6 +81,11 @@ public IRubyObject time(final ThreadContext context, return doTime(context, namespace, key, block); } + @JRubyMethod(name = "register") + public IRubyObject register(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final Block metricSupplier) { + return doRegister(context, namespace, key, metricSupplier); + } + protected abstract IRubyObject doDecrement(ThreadContext context, IRubyObject[] args); protected abstract IRubyObject doIncrement(ThreadContext context, IRubyObject[] args); @@ -88,4 +100,6 @@ protected abstract IRubyObject doReportTime(ThreadContext context, IRubyObject n protected abstract IRubyObject doTime(ThreadContext context, IRubyObject namespace, IRubyObject key, Block block); + + protected abstract IRubyObject doRegister(ThreadContext context, IRubyObject namespace, IRubyObject key, Block metricSupplier); } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java index 1303e1a753..4b1ef12e1d 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java @@ -44,15 +44,15 @@ public final class MetricExt extends AbstractSimpleMetricExt { private static final long serialVersionUID = 1L; - public static final RubySymbol COUNTER = RubyUtil.RUBY.newSymbol("counter"); + // These two metric type symbols need to be package-private because used in NamespacedMetricExt + static final RubySymbol COUNTER = RubyUtil.RUBY.newSymbol("counter"); + static final RubySymbol GAUGE = RubyUtil.RUBY.newSymbol("gauge"); private static final RubyFixnum ONE = RubyUtil.RUBY.newFixnum(1); private static final RubySymbol INCREMENT = RubyUtil.RUBY.newSymbol("increment"); private static final RubySymbol DECREMENT = RubyUtil.RUBY.newSymbol("decrement"); - - private static final RubySymbol GAUGE = RubyUtil.RUBY.newSymbol("gauge"); private static final RubySymbol TIMER = RubyUtil.RUBY.newSymbol("timer"); private static final RubySymbol SET = RubyUtil.RUBY.newSymbol("set"); private static final RubySymbol GET = RubyUtil.RUBY.newSymbol("get"); @@ -153,6 +153,12 @@ protected IRubyObject getTimer(final ThreadContext context, ); } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject namespace, IRubyObject key, Block supplier) { + MetricExt.validateKey(context, null, key); + return collector.callMethod(context, "register", new IRubyObject[]{normalizeNamespace(namespace), key}, supplier); + } + @Override protected IRubyObject doReportTime(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final IRubyObject duration) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java index 2730ab8374..e304351728 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java @@ -116,4 +116,21 @@ private MetricKeys() { public static final RubySymbol WRITES_IN_KEY = RubyUtil.RUBY.newSymbol("writes_in"); + // Batch metrics keys + public static final RubySymbol BATCH_EVENT_COUNT_KEY = RubyUtil.RUBY.newSymbol("event_count"); + + public static final RubySymbol BATCH_AVERAGE_KEY = RubyUtil.RUBY.newSymbol("average"); + + public static final RubySymbol BATCH_KEY = RubyUtil.RUBY.newSymbol("batch"); + + public static final RubySymbol BATCH_COUNT = RubyUtil.RUBY.newSymbol("count"); + + public static final RubySymbol BATCH_TOTAL_EVENTS = RubyUtil.RUBY.newSymbol("total_events"); + + public static final RubySymbol BATCH_TOTAL_BYTES = RubyUtil.RUBY.newSymbol("total_bytes"); + + public static final RubySymbol BATCH_BYTE_SIZE_KEY = RubyUtil.RUBY.newSymbol("byte_size"); + + public static final RubySymbol BATCH_CURRENT_KEY = RubyUtil.RUBY.newSymbol("current"); + } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java index 0ca15cba31..de8708f1b6 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java @@ -77,6 +77,11 @@ public enum MetricType { * A flow-rate {@link FlowMetric}, instantiated with one or more backing {@link Metric}{@code }. */ FLOW_RATE("flow/rate"), + + /** + * A user metric + */ + USER("user"), ; private final String type; diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java index 8a77b4e2f0..38952fc854 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java @@ -53,6 +53,11 @@ public NamespacedMetricExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier) { + return metric.register(context, namespaceName, key, metricSupplier); + } + @JRubyMethod(visibility = Visibility.PRIVATE) public NamespacedMetricExt initialize(final ThreadContext context, final IRubyObject metric, final IRubyObject namespaceName) { @@ -76,7 +81,10 @@ protected IRubyObject getCounter(final ThreadContext context, final IRubyObject @Override protected IRubyObject getGauge(final ThreadContext context, final IRubyObject key, final IRubyObject value) { - return metric.gauge(context, namespaceName, key, value); + metric.gauge(context, namespaceName, key, value); + return collector(context).callMethod( + context, "get", new IRubyObject[]{namespaceName, key, MetricExt.GAUGE} + ); } @Override diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java index c004cc20b8..3a3c283377 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java @@ -111,6 +111,11 @@ protected IRubyObject doTime(final ThreadContext context, final IRubyObject name return block.call(context); } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject namespace, IRubyObject key, Block metricSupplier) { + return context.nil; + } + @Override protected AbstractNamespacedMetricExt createNamespaced(final ThreadContext context, final IRubyObject name) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java index 152541b227..21eb5c0186 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java @@ -20,6 +20,7 @@ package org.logstash.instrument.metrics; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyClass; @@ -31,6 +32,7 @@ import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; +import org.logstash.plugins.NamespacedMetricImpl; @JRubyClass(name = "NamespacedNullMetric", parent = "AbstractNamespacedMetric") public final class NullNamespacedMetricExt extends AbstractNamespacedMetricExt { @@ -69,6 +71,11 @@ public NullNamespacedMetricExt initialize(final ThreadContext context, return this; } + @Override + public Metric asApiMetric() { + return NamespacedMetricImpl.getNullMetric(); + } + @Override protected IRubyObject getCollector(final ThreadContext context) { return metric.collector(context); @@ -112,6 +119,11 @@ protected IRubyObject doReportTime(final ThreadContext context, final IRubyObjec return context.nil; } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier) { + return context.nil; + } + @Override @SuppressWarnings("rawtypes") protected RubyArray getNamespaceName(final ThreadContext context) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/UserMetric.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/UserMetric.java new file mode 100644 index 0000000000..4c015efb57 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/UserMetric.java @@ -0,0 +1,42 @@ +package org.logstash.instrument.metrics; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jruby.RubySymbol; +import org.jruby.runtime.Block; +import org.jruby.runtime.JavaInternalBlockBody; +import org.jruby.runtime.Signature; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; + +public class UserMetric { + private UserMetric() {} + + private static Logger LOGGER = LogManager.getLogger(UserMetric.class); + + public static > USER_METRIC fromRubyBase( + final AbstractNamespacedMetricExt metric, + final RubySymbol key, + final co.elastic.logstash.api.UserMetric.Factory metricFactory + ) { + final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); + + final Block metricSupplier = new Block(new JavaInternalBlockBody(context.runtime, Signature.NO_ARGUMENTS) { + @Override + public IRubyObject yield(ThreadContext threadContext, IRubyObject[] iRubyObjects) { + return RubyUtil.toRubyObject(metricFactory.create(key.asJavaString())); + } + }); + + final IRubyObject result = metric.register(context, key, metricSupplier); + final Class type = metricFactory.getType(); + if (!type.isAssignableFrom(result.getJavaClass())) { + LOGGER.warn("UserMetric type mismatch for %s (expected: %s, received: %s); " + + "a null implementation will be substituted", key.asJavaString(), type, result.getJavaClass()); + return metricFactory.nullImplementation(); + } + + return result.toJava(type); + } +} diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/gauge/LazyDelegatingGauge.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/gauge/LazyDelegatingGauge.java index cf9f30ad8b..10c10fceef 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/gauge/LazyDelegatingGauge.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/gauge/LazyDelegatingGauge.java @@ -23,10 +23,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jruby.RubyHash; +import org.jruby.RubySymbol; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; import org.logstash.ext.JrubyTimestampExtLibrary.RubyTimestamp; import org.logstash.instrument.metrics.AbstractMetric; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; import org.logstash.instrument.metrics.MetricType; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -39,11 +45,27 @@ public class LazyDelegatingGauge extends AbstractMetric implements Gauge private static final Logger LOGGER = LogManager.getLogger(LazyDelegatingGauge.class); + public static final LazyDelegatingGauge DUMMY_GAUGE = new LazyDelegatingGauge("dummy"); + protected final String key; @SuppressWarnings("rawtypes") private GaugeMetric lazyMetric; + + public static LazyDelegatingGauge fromRubyBase(final AbstractNamespacedMetricExt metric, final RubySymbol key) { + final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); + // just initialize an empty gauge + final IRubyObject gauge = metric.gauge(context, key, context.runtime.newArray(context.runtime.newString("undefined"), context.runtime.newString("undefined"))); + final LazyDelegatingGauge javaGauge; + if (LazyDelegatingGauge.class.isAssignableFrom(gauge.getJavaClass())) { + javaGauge = gauge.toJava(LazyDelegatingGauge.class); + } else { + javaGauge = DUMMY_GAUGE; + } + return javaGauge; + } + /** * Constructor - null initial value * diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/ConcurrentLiveTimerMetric.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/ConcurrentLiveTimerMetric.java index 4adbef05a0..9b681944b5 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/ConcurrentLiveTimerMetric.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/ConcurrentLiveTimerMetric.java @@ -43,10 +43,13 @@ protected ConcurrentLiveTimerMetric(final String name) { @Override public T time(ExceptionalSupplier exceptionalSupplier) throws E { try { - trackedMillisState.getAndUpdate(TrackedMillisState::withIncrementedConcurrency); + trackedMillisState.getAndUpdate(existing -> existing.withIncrementedConcurrency(nanoTimeSupplier.getAsLong())); return exceptionalSupplier.get(); } finally { - trackedMillisState.getAndUpdate(TrackedMillisState::withDecrementedConcurrency); + // lock in the actual time completed, and resolve state separately + // so that contention for recording state is not included in measurement. + final long endTime = nanoTimeSupplier.getAsLong(); + trackedMillisState.getAndUpdate(existing -> existing.withDecrementedConcurrency(endTime)); } } @@ -65,13 +68,13 @@ private long getUntrackedMillis() { } private long getTrackedMillis() { - return this.trackedMillisState.getAcquire().getValue(); + return this.trackedMillisState.getAcquire().getValue(nanoTimeSupplier.getAsLong()); } interface TrackedMillisState { - TrackedMillisState withIncrementedConcurrency(); - TrackedMillisState withDecrementedConcurrency(); - long getValue(); + TrackedMillisState withIncrementedConcurrency(long asOfNanoTime); + TrackedMillisState withDecrementedConcurrency(long asOfNanoTime); + long getValue(long asOfNanoTime); } private class StaticTrackedMillisState implements TrackedMillisState { @@ -89,18 +92,18 @@ public StaticTrackedMillisState() { } @Override - public TrackedMillisState withIncrementedConcurrency() { - return new DynamicTrackedMillisState(nanoTimeSupplier.getAsLong(), this.cumulativeMillis, this.excessNanos, 1); + public TrackedMillisState withIncrementedConcurrency(final long asOfNanoTime) { + return new DynamicTrackedMillisState(asOfNanoTime, this.cumulativeMillis, this.excessNanos, 1); } @Override - public TrackedMillisState withDecrementedConcurrency() { + public TrackedMillisState withDecrementedConcurrency(final long asOfNanoTime) { throw new IllegalStateException("TimerMetrics cannot track negative concurrency"); } @Override - public long getValue() { + public long getValue(final long asOfNanoTime) { return cumulativeMillis; } } @@ -122,26 +125,26 @@ private class DynamicTrackedMillisState implements TrackedMillisState { } @Override - public TrackedMillisState withIncrementedConcurrency() { - return withAdjustedConcurrency(Vector.INCREMENT); + public TrackedMillisState withIncrementedConcurrency(final long asOfNanoTime) { + return withAdjustedConcurrency(asOfNanoTime, Vector.INCREMENT); } @Override - public TrackedMillisState withDecrementedConcurrency() { - return withAdjustedConcurrency(Vector.DECREMENT); + public TrackedMillisState withDecrementedConcurrency(final long asOfNanoTime) { + return withAdjustedConcurrency(asOfNanoTime, Vector.DECREMENT); } @Override - public long getValue() { - final long nanoAdjustment = getNanoAdjustment(nanoTimeSupplier.getAsLong()); + public long getValue(final long asOfNanoTime) { + final long nanoAdjustment = getNanoAdjustment(asOfNanoTime); final long milliAdjustment = wholeMillisFromNanos(nanoAdjustment); return Math.addExact(this.millisAtCheckpoint, milliAdjustment); } - private TrackedMillisState withAdjustedConcurrency(final Vector concurrencyAdjustmentVector) { + private TrackedMillisState withAdjustedConcurrency(final long asOfNanoTime, final Vector concurrencyAdjustmentVector) { final int newConcurrency = Math.addExact(this.concurrencySinceCheckpoint, concurrencyAdjustmentVector.value()); - final long newCheckpointNanoTime = nanoTimeSupplier.getAsLong(); + final long newCheckpointNanoTime = asOfNanoTime; final long totalNanoAdjustment = getNanoAdjustment(newCheckpointNanoTime); @@ -165,7 +168,7 @@ private long getNanoAdjustment(final long checkpointNanoTime) { /** * This private enum is a type-safety guard for - * {@link DynamicTrackedMillisState#withAdjustedConcurrency(Vector)}. + * {@link DynamicTrackedMillisState#withAdjustedConcurrency(long, Vector)}. */ private enum Vector { INCREMENT{ int value() { return +1; } }, diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/TimerMetricFactory.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/TimerMetricFactory.java index d2cb39be9e..04af5f5f79 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/TimerMetricFactory.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/TimerMetricFactory.java @@ -5,6 +5,10 @@ public class TimerMetricFactory { static final TimerMetricFactory INSTANCE = new TimerMetricFactory(); + public static TimerMetricFactory getInstance() { + return INSTANCE; + } + private TimerMetricFactory() { } diff --git a/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java b/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java index bb61ac12a1..cc57421e08 100644 --- a/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java +++ b/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java @@ -23,17 +23,23 @@ import co.elastic.logstash.api.CounterMetric; import co.elastic.logstash.api.Metric; import co.elastic.logstash.api.NamespacedMetric; +import co.elastic.logstash.api.UserMetric; +import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyObject; import org.jruby.RubySymbol; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; import org.logstash.Rubyfier; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.NullMetricExt; +import org.logstash.instrument.metrics.NullNamespacedMetricExt; import org.logstash.instrument.metrics.timer.TimerMetric; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.function.Supplier; import java.util.stream.Stream; @@ -43,6 +49,25 @@ */ public class NamespacedMetricImpl implements NamespacedMetric { + private static final NamespacedMetric NULL_METRIC; + static { + final Ruby rubyRuntime = RubyUtil.RUBY; + final ThreadContext context = rubyRuntime.getCurrentContext(); + final NullMetricExt nullMetricExt = NullMetricExt.create(); + final AbstractNamespacedMetricExt namespacedMetricExt = NullNamespacedMetricExt.create(nullMetricExt, rubyRuntime.newArray()); + + NULL_METRIC = new NamespacedMetricImpl(context, namespacedMetricExt){ + @Override + public NamespacedMetric namespace(String... key) { + return this; + } + }; + } + + public static NamespacedMetric getNullMetric() { + return NULL_METRIC; + } + private final ThreadContext threadContext; private final AbstractNamespacedMetricExt metrics; @@ -67,6 +92,13 @@ public co.elastic.logstash.api.TimerMetric timer(final String metric) { return TimerMetric.fromRubyBase(metrics, threadContext.getRuntime().newString(metric).intern()); } + @Override + public > USER_METRIC register(String metric, UserMetric.Factory userMetricFactory) { + USER_METRIC userMetric = org.logstash.instrument.metrics.UserMetric.fromRubyBase(metrics, threadContext.runtime.newSymbol(metric), userMetricFactory); + + return Objects.requireNonNullElseGet(userMetric, userMetricFactory::nullImplementation); + } + @Override public NamespacedMetric namespace(final String... key) { final IRubyObject[] rubyfiedKeys = Stream.of(key) diff --git a/logstash-core/src/main/java/org/logstash/settings/PasswordSetting.java b/logstash-core/src/main/java/org/logstash/settings/PasswordSetting.java new file mode 100644 index 0000000000..d0329f663b --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/settings/PasswordSetting.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.logstash.settings; + +import co.elastic.logstash.api.Password; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +public class PasswordSetting extends Coercible { + + private static final Logger LOG = LogManager.getLogger(); + + public PasswordSetting(String name, Object defaultValue) { + this(name, defaultValue, true); + } + + public PasswordSetting(String name, Object defaultValue, boolean strict) { + super(name, defaultValue, strict, noValidator()); + } + + @Override + public Password coerce(Object obj) { + if (obj instanceof Password) { + return (Password) obj; + } + if (obj != null && !(obj instanceof String)) { + throw new IllegalArgumentException("Setting `" + getName() + "` could not coerce non-string value to password"); + } + return new Password((String) obj); + } + + public Logger getLogger() { + return LOG; + } +} diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/AtomicIORatioMetricTest.java b/logstash-core/src/test/java/org/logstash/ackedqueue/AtomicIORatioMetricTest.java new file mode 100644 index 0000000000..2d209d2b1c --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/AtomicIORatioMetricTest.java @@ -0,0 +1,129 @@ +package org.logstash.ackedqueue; + +import org.apache.logging.log4j.Logger; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.stringContainsInOrder; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; + +public class AtomicIORatioMetricTest { + @Test + public void test() { + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name"); + + assertThat(ioRatioMetric.getValue()).isNaN(); + + ioRatioMetric.incrementBy(1024, 768); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.75); + + ioRatioMetric.incrementBy(256, 128); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.7); + + ioRatioMetric.incrementBy(512, 128); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.5714); + + ioRatioMetric.incrementBy(256, 0); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.5); + + ioRatioMetric.incrementBy(0, 1024); + assertThat(ioRatioMetric.getValue()).isEqualTo(1.0); + + ioRatioMetric.reset(); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(0L); + assertThat(value.bytesOut()).isEqualTo(0L); + }); + + int iterations = 100000000; + int bytesInPerIteration = 4000000; + int bytesOutPerIteration = 3000000; + for (int i = 0; i < iterations; i++) { + ioRatioMetric.incrementBy(bytesInPerIteration, bytesOutPerIteration); + } + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(Math.multiplyExact((long)bytesInPerIteration, iterations)); + assertThat(value.bytesOut()).isEqualTo(Math.multiplyExact((long)bytesOutPerIteration, iterations)); + }); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.75); + } + + @Test + public void testZeroBytesInPositiveBytesOut() { + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name"); + + ioRatioMetric.incrementBy(0, 768); + assertThat(ioRatioMetric.getValue()).isEqualTo(Double.POSITIVE_INFINITY); + } + + @Test + public void testNegativeBytesIn() { + final Logger mockLogger = Mockito.mock(Logger.class); + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name", mockLogger); + ioRatioMetric.incrementBy(-1, 768); + + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("cannot decrement")), eq(ioRatioMetric.getName())); + } + + @Test + public void testNegativeBytesOut() { + final Logger mockLogger = Mockito.mock(Logger.class); + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name", mockLogger); + ioRatioMetric.incrementBy(768, -1); + + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("cannot decrement")), eq(ioRatioMetric.getName())); + } + + @Test + public void testZeroBytesInZeroBytesOut() { + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name"); + + ioRatioMetric.incrementBy(0, 0); + assertThat(ioRatioMetric.getValue()).isNaN(); + } + + @Test + public void testLongBytesInOverflow() { + final Logger mockLogger = Mockito.mock(Logger.class); + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name", mockLogger); + ioRatioMetric.setTo(Long.MAX_VALUE, 2L); + + assertThat(ioRatioMetric.getValue()).isEqualTo(2.168E-19); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(Long.MAX_VALUE); + assertThat(value.bytesOut()).isEqualTo(2L); + }); + + //overflow reset + ioRatioMetric.incrementBy(1, 10); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(4611686018427387903L + 1L); + assertThat(value.bytesOut()).isEqualTo(1L + 10L); + }); + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("long overflow", "precision", "reduced"))); + } + + @Test + public void testLongBytesOutOverflow() { + final Logger mockLogger = Mockito.mock(Logger.class); + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name", mockLogger); + ioRatioMetric.setTo(2L, Long.MAX_VALUE); + + assertThat(ioRatioMetric.getValue()).isEqualTo(4.612E18); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(2L); + assertThat(value.bytesOut()).isEqualTo(Long.MAX_VALUE); + }); + + //overflow reset/truncate + ioRatioMetric.incrementBy(10, 1); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(1L + 10L); + assertThat(value.bytesOut()).isEqualTo(4611686018427387903L + 1L); + }); + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("long overflow", "precision", "reduced"))); + } + +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetricTest.java b/logstash-core/src/test/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetricTest.java new file mode 100644 index 0000000000..fe5067e7b3 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetricTest.java @@ -0,0 +1,47 @@ +package org.logstash.ackedqueue; + +import org.junit.Test; +import org.logstash.instrument.metrics.ManualAdvanceClock; +import org.logstash.instrument.metrics.UptimeMetric; +import org.logstash.instrument.metrics.timer.TestTimerMetricFactory; +import org.logstash.instrument.metrics.timer.TimerMetric; + +import java.time.Duration; +import java.time.Instant; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.logstash.instrument.metrics.VisibilityUtil.createUptimeMetric; + +public class CalculatedRelativeSpendMetricTest { + @Test + public void testCalculateRelativeSpendMetric() { + final ManualAdvanceClock clock = new ManualAdvanceClock(Instant.now()); + final TimerMetric burnMetric = new TestTimerMetricFactory(clock::nanoTime).newTimerMetric("burn"); + final UptimeMetric uptimeMetric = createUptimeMetric("wall", clock::nanoTime); + final CalculatedRelativeSpendMetric relativeSpendMetric = new CalculatedRelativeSpendMetric("spend", burnMetric, uptimeMetric); + + clock.advance(Duration.ofMillis(17)); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.0); + + relativeSpendMetric.time(() -> clock.advance(Duration.ofMillis(17))); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.5); + + clock.advance(Duration.ofMillis(34)); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.25); + + relativeSpendMetric.time(() -> clock.advance(Duration.ofMillis(147))); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.7628); + + // nesting for forced concurrency; interleaving validated upstream in TimerMetric + relativeSpendMetric.time(() -> { + clock.advance(Duration.ofMillis(149)); + relativeSpendMetric.time(() -> clock.advance(Duration.ofMillis(842))); + clock.advance(Duration.ofMillis(17)); + }); + assertThat(relativeSpendMetric.getValue()).isEqualTo(1.647); + + // advance wall clock without any new timings + clock.advance(Duration.ofMillis(6833)); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.25); + } +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java b/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java new file mode 100644 index 0000000000..3035507f95 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java @@ -0,0 +1,239 @@ +package org.logstash.ackedqueue; + +import com.github.luben.zstd.Zstd; +import org.apache.logging.log4j.Logger; +import org.junit.Test; +import org.mockito.Mockito; + +import java.security.SecureRandom; +import java.util.Arrays; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertThrows; +import static org.mockito.Matchers.argThat; + +public class CompressionCodecTest { + static final ImmutableByteArrayBarrier RAW_BYTES = new ImmutableByteArrayBarrier(( + "this is a string of text with repeated substrings that is designed to be "+ + "able to be compressed into a string that is smaller than the original input "+ + "so that we can assert that the compression codecs compress strings to be "+ + "smaller than their uncompressed representations").getBytes()); + static final ImmutableByteArrayBarrier COMPRESSED_MINIMAL = new ImmutableByteArrayBarrier(compress(RAW_BYTES.bytes(), -1)); + static final ImmutableByteArrayBarrier COMPRESSED_DEFAULT = new ImmutableByteArrayBarrier(compress(RAW_BYTES.bytes(), 3)); + static final ImmutableByteArrayBarrier COMPRESSED_MAXIMUM = new ImmutableByteArrayBarrier(compress(RAW_BYTES.bytes(), 22)); + + private final CompressionCodec codecDisabled = CompressionCodec.fromConfigValue("disabled").create(); + private final CompressionCodec codecNone = CompressionCodec.fromConfigValue("none").create(); + private final CompressionCodec codecSpeed = CompressionCodec.fromConfigValue("speed").create(); + private final CompressionCodec codecBalanced = CompressionCodec.fromConfigValue("balanced").create(); + private final CompressionCodec codecSize = CompressionCodec.fromConfigValue("size").create(); + + @Test + public void testDisabledCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("disabled").create(); + assertDecodesRaw(compressionCodec); + + // ensure true pass-through when compression is disabled, even if the payload looks like ZSTD + assertThat(compressionCodec.decode(COMPRESSED_MINIMAL.bytes()), is(equalTo(COMPRESSED_MINIMAL.bytes()))); + assertThat(compressionCodec.decode(COMPRESSED_DEFAULT.bytes()), is(equalTo(COMPRESSED_DEFAULT.bytes()))); + assertThat(compressionCodec.decode(COMPRESSED_MAXIMUM.bytes()), is(equalTo(COMPRESSED_MAXIMUM.bytes()))); + } + + @Test + public void testDisabledCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("disabled").create(); + // ensure true pass-through when compression is disabled + assertThat(compressionCodec.encode(RAW_BYTES.bytes()), is(equalTo(RAW_BYTES.bytes()))); + } + + @Test + public void testDisabledCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("disabled", mockLogger).create(); + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("compression support", "disabled"))); + } + + @Test + public void testNoneCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("none").create(); + assertDecodesRaw(compressionCodec); + assertDecodesDeflateAnyLevel(compressionCodec); + assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); + } + + @Test + public void testNoneCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("none").create(); + assertThat(compressionCodec.encode(RAW_BYTES.bytes()), is(equalTo(RAW_BYTES.bytes()))); + } + + @Test + public void testNoneCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("none", mockLogger).create(); + Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "read-only"))); + } + + @Test + public void testSpeedCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("speed").create(); + assertDecodesRaw(compressionCodec); + assertDecodesDeflateAnyLevel(compressionCodec); + assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); + } + + @Test + public void testSpeedCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("speed").create(); + assertEncodesSmallerRoundTrip(compressionCodec); + } + + @Test + public void testSpeedCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("speed", mockLogger).create(); + Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "speed"))); + } + + @Test + public void testBalancedCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("balanced").create(); + assertDecodesRaw(compressionCodec); + assertDecodesDeflateAnyLevel(compressionCodec); + assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); + } + + @Test + public void testBalancedCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("balanced").create(); + assertEncodesSmallerRoundTrip(compressionCodec); + } + + @Test + public void testBalancedCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("balanced", mockLogger).create(); + Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "balanced"))); + } + + @Test + public void testSizeCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("size").create(); + assertDecodesRaw(compressionCodec); + assertDecodesDeflateAnyLevel(compressionCodec); + assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); + } + + @Test + public void testSizeCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("size").create(); + assertEncodesSmallerRoundTrip(compressionCodec); + } + + @Test + public void testSizeCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("size", mockLogger).create(); + Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "size"))); + } + + @Test(timeout=1000) + public void testCompressionCodecDecodeTailTruncated() throws Exception { + final byte[] truncatedInput = copyWithTruncatedTail(COMPRESSED_DEFAULT.bytes(), 32); + + final RuntimeException thrownException = assertThrows(RuntimeException.class, () -> codecNone.decode(truncatedInput)); + assertThat(thrownException.getMessage(), containsString("Exception while decoding")); + final Throwable rootCause = extractRootCause(thrownException); + assertThat(rootCause.getMessage(), containsString("Data corruption detected")); + } + + byte[] copyWithTruncatedTail(final byte[] input, final int tailSize) { + int startIndex = (input.length < tailSize) ? 0 : input.length - tailSize; + + final byte[] result = Arrays.copyOf(input, input.length); + Arrays.fill(result, startIndex, result.length, (byte) 0); + + return result; + } + + @Test(timeout=1000) + public void testCompressionCodecDecodeTailScrambled() throws Exception { + final byte[] scrambledInput = copyWithScrambledTail(COMPRESSED_DEFAULT.bytes(), 32); + + final RuntimeException thrownException = assertThrows(RuntimeException.class, () -> codecNone.decode(scrambledInput)); + assertThat(thrownException.getMessage(), containsString("Exception while decoding")); + final Throwable rootCause = extractRootCause(thrownException); + assertThat(rootCause.getMessage(), anyOf(containsString("Data corruption detected"), containsString("Destination buffer is too small"))); + } + + byte[] copyWithScrambledTail(final byte[] input, final int tailSize) { + final SecureRandom secureRandom = new SecureRandom(); + int startIndex = (input.length < tailSize) ? 0 : input.length - tailSize; + + byte[] randomBytes = new byte[input.length - startIndex]; + secureRandom.nextBytes(randomBytes); + + final byte[] result = Arrays.copyOf(input, input.length); + System.arraycopy(randomBytes, 0, result, startIndex, randomBytes.length); + + return result; + } + + @Test(timeout=1000) + public void testCompressionDecodeTailNullPadded() throws Exception { + final byte[] nullPaddedInput = copyWithNullPaddedTail(COMPRESSED_DEFAULT.bytes(), 32); + + final RuntimeException thrownException = assertThrows(RuntimeException.class, () -> codecNone.decode(nullPaddedInput)); + assertThat(thrownException.getMessage(), containsString("Exception while decoding")); + final Throwable rootCause = extractRootCause(thrownException); + assertThat(rootCause.getMessage(), anyOf(containsString("Unknown frame descriptor"), containsString("Data corruption detected"))); + } + + byte[] copyWithNullPaddedTail(final byte[] input, final int tailSize) { + return Arrays.copyOf(input, Math.addExact(input.length, tailSize)); + } + + Throwable extractRootCause(final Throwable throwable) { + Throwable current; + Throwable cause = throwable; + do { + current = cause; + cause = current.getCause(); + } while (cause != null && cause != current); + return current; + } + + void assertDecodesRaw(final CompressionCodec codec) { + assertThat(codec.decode(RAW_BYTES.bytes()), is(equalTo(RAW_BYTES.bytes()))); + } + + void assertDecodesDeflateAnyLevel(final CompressionCodec codec) { + // zstd levels range from -7 to 22. + for (int level = -7; level < 22; level++) { + final byte[] compressed = compress(RAW_BYTES.bytes(), level); + assertThat(String.format("zstd level %s (%s bytes)", level, compressed.length), codec.decode(compressed), is(equalTo(RAW_BYTES.bytes()))); + } + } + + void assertDecodesOutputOfAllKnownCompressionCodecs(final CompressionCodec codec) { + assertThat(codec.decode(codecDisabled.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + assertThat(codec.decode(codecNone.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + assertThat(codec.decode(codecSpeed.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + assertThat(codec.decode(codecBalanced.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + assertThat(codec.decode(codecSize.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + } + + void assertEncodesSmallerRoundTrip(final CompressionCodec codec) { + final byte[] input = RAW_BYTES.bytes(); + + final byte[] encoded = codec.encode(input); + assertThat("encoded is smaller", encoded.length, is(lessThan(input.length))); + assertThat("shaped like zstd", AbstractZstdAwareCompressionCodec.isZstd(encoded), is(true)); + assertThat("round trip decode", codec.decode(encoded), is(equalTo(input))); + } + + public static byte[] compress(byte[] input, int level) { + return Zstd.compress(input, level); + } +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/ImmutableByteArrayBarrier.java b/logstash-core/src/test/java/org/logstash/ackedqueue/ImmutableByteArrayBarrier.java new file mode 100644 index 0000000000..276e4d97db --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/ImmutableByteArrayBarrier.java @@ -0,0 +1,20 @@ +package org.logstash.ackedqueue; + +import java.util.Arrays; + +/** + * An {@link ImmutableByteArrayBarrier} provides an immutability shield around a {@code byte[]}. + * It stores an inaccessible copy of the provided {@code byte[]}, and makes copies of that copy + * available via {@link ImmutableByteArrayBarrier#bytes}. + * @param bytes the byte array + */ +public record ImmutableByteArrayBarrier(byte[] bytes) { + public ImmutableByteArrayBarrier(byte[] bytes) { + this.bytes = Arrays.copyOf(bytes, bytes.length); + } + + @Override + public byte[] bytes() { + return Arrays.copyOf(bytes, bytes.length); + } +} diff --git a/logstash-core/src/test/java/org/logstash/execution/QueueReadClientBatchMetricsTest.java b/logstash-core/src/test/java/org/logstash/execution/QueueReadClientBatchMetricsTest.java new file mode 100644 index 0000000000..588064e41a --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/execution/QueueReadClientBatchMetricsTest.java @@ -0,0 +1,133 @@ +package org.logstash.execution; + +import static org.hamcrest.MatcherAssert.assertThat; +import org.jruby.RubyArray; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.junit.Before; +import org.junit.Test; +import org.logstash.Event; +import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; +import org.logstash.ext.JrubyEventExtLibrary; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MetricKeys; +import org.logstash.instrument.metrics.MockNamespacedMetric; +import org.logstash.instrument.metrics.counter.LongCounter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class QueueReadClientBatchMetricsTest { + + public static final class MockQueueBatch implements QueueBatch { + + private final long processingTimeNanos; + private final List events; + + public MockQueueBatch(long processingTimeNanos, JrubyEventExtLibrary.RubyEvent... events) { + this.processingTimeNanos = processingTimeNanos; + this.events = Arrays.stream(events).toList(); + } + + @Override + @SuppressWarnings("unchecked") + public RubyArray to_a() { + List list = new ArrayList<>(events); + return (RubyArray) RubyUtil.RUBY.newArray(list); + } + + @Override + @SuppressWarnings("unchecked") + public Collection events() { + return to_a(); + } + + @Override + public void close() throws IOException { + // no-op + } + + @Override + public int filteredSize() { + return events.size(); + } + + public long getProcessingTimeNanos() { + return processingTimeNanos; + } + } + + private AbstractNamespacedMetricExt metric; + private QueueReadClientBatchMetrics sut; + private LongCounter batchCounter; + private LongCounter batchByteSizeCounter; + private JrubyEventExtLibrary.RubyEvent rubyEvent; + + @Before + public void setUp() { + metric = MockNamespacedMetric.create(); + sut = new QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode.FULL); + sut.setupMetrics(metric); + + ThreadContext context = metric.getRuntime().getCurrentContext(); + batchCounter = LongCounter.fromRubyBase(metric.namespace(context, MetricKeys.BATCH_KEY), MetricKeys.BATCH_COUNT); + batchByteSizeCounter = LongCounter.fromRubyBase(metric.namespace(context, MetricKeys.BATCH_KEY), MetricKeys.BATCH_TOTAL_BYTES); + + rubyEvent = JrubyEventExtLibrary.RubyEvent.newRubyEvent(RubyUtil.RUBY, new Event()); + } + + @Test + public void givenEmptyBatchAndFullMetricsWhenUpdateBatchMetricsThenNoMetricsAreUpdated() { + QueueBatch emptyBatch = new MockQueueBatch(10); + + sut.updateBatchMetrics(emptyBatch); + + assertEquals(0L, batchCounter.getValue().longValue()); + } + + @Test + public void givenNonEmptyBatchAndFullMetricsWhenUpdateBatchMetricsThenMetricsAreUpdated() { + QueueBatch batch = new MockQueueBatch(10, rubyEvent); + final long expectedBatchByteSize = rubyEvent.getEvent().estimateMemory(); + + sut.updateBatchMetrics(batch); + + assertEquals(1L, batchCounter.getValue().longValue()); + assertEquals(expectedBatchByteSize, batchByteSizeCounter.getValue().longValue()); + } + + @Test + public void givenNonEmptyBatchesAndMinimalMetricsThenMetricsAreUpdated() { + sut = new QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode.MINIMAL); + sut.setupMetrics(metric); + + QueueBatch batch = new MockQueueBatch(10, rubyEvent); + final long expectedBatchByteSize = rubyEvent.getEvent().estimateMemory(); + + for (int i = 0; i < 200; i++) { + sut.updateBatchMetrics(batch); + } + sut.updateBatchMetrics(batch); + + assertThat(batchCounter.getValue(), org.hamcrest.Matchers.greaterThan(1L)); + assertThat(batchByteSizeCounter.getValue(), org.hamcrest.Matchers.greaterThan(expectedBatchByteSize)); + } + + @Test + public void givenNonEmptyQueueWhenBatchIsReadAndMetricIsDisabledThenBatchCounterMetricIsNotUpdated() { + sut = new QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode.DISABLED); + sut.setupMetrics(metric); + QueueBatch batch = new MockQueueBatch(10, rubyEvent); + + sut.updateBatchMetrics(batch); + + assertEquals(0L, batchCounter.getValue().longValue()); + assertEquals(0L, batchByteSizeCounter.getValue().longValue()); + } +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java index dff3c4845a..386aa628f2 100644 --- a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java +++ b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java @@ -25,26 +25,50 @@ import java.util.concurrent.BlockingQueue; import org.jruby.RubyHash; import org.jruby.runtime.ThreadContext; +import org.junit.Before; import org.junit.Test; +import org.logstash.Event; import org.logstash.RubyTestBase; +import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.execution.QueueBatch; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MetricKeys; +import org.logstash.instrument.metrics.MockNamespacedMetric; +import org.logstash.instrument.metrics.counter.LongCounter; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; /** * Tests for {@link JrubyMemoryReadClientExt}. */ public final class JrubyMemoryReadClientExtTest extends RubyTestBase { + private JrubyEventExtLibrary.RubyEvent testEvent; + private BlockingQueue queue; + private AbstractNamespacedMetricExt metric; + private LongCounter batchCounter; + private LongCounter batchByteSizeCounter; + + @Before + public void setUp() { + testEvent = JrubyEventExtLibrary.RubyEvent.newRubyEvent(RubyUtil.RUBY, new Event()); + queue = new ArrayBlockingQueue<>(10); + metric = MockNamespacedMetric.create(); + ThreadContext context = metric.getRuntime().getCurrentContext(); + batchCounter = LongCounter.fromRubyBase(metric.namespace(context, MetricKeys.BATCH_KEY), MetricKeys.BATCH_COUNT); + batchByteSizeCounter = LongCounter.fromRubyBase(metric.namespace(context, MetricKeys.BATCH_KEY), MetricKeys.BATCH_TOTAL_BYTES); + } + @Test @SuppressWarnings("deprecation") public void testInflightBatchesTracking() throws InterruptedException, IOException { - final BlockingQueue queue = - new ArrayBlockingQueue<>(10); final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50); final ThreadContext context = client.getRuntime().getCurrentContext(); + client.setPipelineMetric(metric); final QueueBatch batch = client.readBatch(); final RubyHash inflight = client.rubyGetInflightBatches(context); assertThat(inflight.size(), is(1)); @@ -53,4 +77,55 @@ public void testInflightBatchesTracking() throws InterruptedException, IOExcepti client.closeBatch(batch); assertThat(client.rubyGetInflightBatches(context).size(), is(0)); } + + @Test + public void givenNonEmptyQueueWhenBatchIsReadThenBatchCounterMetricIsUpdated() throws InterruptedException { + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, + QueueFactoryExt.BatchMetricMode.FULL); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + assertEquals(1L, batchCounter.getValue().longValue()); + } + + @Test + public void givenNonEmptyQueueWhenBatchIsReadAndMetricIsDisabledThenBatchCounterMetricIsNotUpdated() throws InterruptedException { + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, + QueueFactoryExt.BatchMetricMode.DISABLED); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + assertEquals(0L, batchCounter.getValue().longValue()); + } + + @Test + public void givenEmptyQueueWhenEmptyBatchIsReadAndMetricIsFullyCollectedThenBatchCounterMetricIsNotUpdated() throws InterruptedException { + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, + QueueFactoryExt.BatchMetricMode.FULL); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(0, batch.filteredSize()); + assertEquals(0L, batchCounter.getValue().longValue()); + } + + @Test + public void givenNonEmptyQueueWhenBatchIsReadThenBatchByteSizeMetricIsUpdated() throws InterruptedException { + final long expectedBatchByteSize = testEvent.getEvent().estimateMemory(); + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, + QueueFactoryExt.BatchMetricMode.FULL); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + assertEquals(expectedBatchByteSize, batchByteSizeCounter.getValue().longValue()); + } } diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java index 21c6a7faf8..3c7a562b08 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java @@ -52,6 +52,7 @@ public void ensurePassivity(){ nameMap.put(MetricType.GAUGE_RUBYHASH, "gauge/rubyhash"); nameMap.put(MetricType.GAUGE_RUBYTIMESTAMP, "gauge/rubytimestamp"); nameMap.put(MetricType.FLOW_RATE, "flow/rate"); + nameMap.put(MetricType.USER, "user"); //ensure we are testing all of the enumerations assertThat(EnumSet.allOf(MetricType.class).size()).isEqualTo(nameMap.size()); diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java new file mode 100644 index 0000000000..bf93411d97 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -0,0 +1,109 @@ +package org.logstash.instrument.metrics; + +import org.jruby.Ruby; +import org.jruby.RubyArray; +import org.jruby.RubyClass; +import org.jruby.RubySymbol; +import org.jruby.runtime.Block; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; +import org.logstash.instrument.metrics.counter.LongCounter; +import org.logstash.instrument.metrics.gauge.LazyDelegatingGauge; +import org.logstash.instrument.metrics.timer.TimerMetric; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Trivial implementation of AbstractNamespacedMetricExt where each abstract creation + * metric is implemented by pooling metric instances by name. + * */ +@SuppressWarnings({"rawtypes", "serializable"}) +public class MockNamespacedMetric extends AbstractNamespacedMetricExt { + + private static final long serialVersionUID = -6507123659910450215L; + + private transient final ConcurrentMap metrics = new ConcurrentHashMap<>(); + + public static MockNamespacedMetric create() { + return new MockNamespacedMetric(RubyUtil.RUBY, RubyUtil.NAMESPACED_METRIC_CLASS); + } + + MockNamespacedMetric(final Ruby runtime, final RubyClass metaClass) { + super(runtime, metaClass); + } + + @Override + protected IRubyObject getGauge(ThreadContext context, IRubyObject key, IRubyObject value) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), LazyDelegatingGauge::new)); + } + + @Override + protected RubyArray getNamespaceName(ThreadContext context) { + return null; + } + + @Override + protected IRubyObject getCounter(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), LongCounter::new)); + } + + @Override + protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), TimerMetric::create)); + } + + @Override + protected IRubyObject doTime(ThreadContext context, IRubyObject key, Block block) { + return null; + } + + @Override + protected IRubyObject doReportTime(ThreadContext context, IRubyObject key, IRubyObject duration) { + return null; + } + + @Override + protected IRubyObject doIncrement(ThreadContext context, IRubyObject[] args) { + return null; + } + + @Override + protected IRubyObject doDecrement(ThreadContext context, IRubyObject[] args) { + return null; + } + + @Override + public AbstractMetricExt getMetric() { + return NullMetricExt.create(); + } + + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier) { + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), (k) -> metricSupplier.call(context))); + } + + @Override + protected AbstractNamespacedMetricExt createNamespaced(ThreadContext context, IRubyObject name) { + return this; + } + + @Override + protected IRubyObject getCollector(ThreadContext context) { + return null; + } + + private static void requireRubySymbol(IRubyObject value, String paramName) { + if (!(value instanceof RubySymbol)) { + throw new IllegalArgumentException(paramName + " must be a RubySymbol instead was: " + value.getClass()); + } + } +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/VisibilityUtil.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/VisibilityUtil.java new file mode 100644 index 0000000000..f252fae776 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/VisibilityUtil.java @@ -0,0 +1,13 @@ +package org.logstash.instrument.metrics; + +import org.logstash.instrument.metrics.timer.TimerMetric; + +import java.util.function.LongSupplier; + +public class VisibilityUtil { + private VisibilityUtil() {} + + public static UptimeMetric createUptimeMetric(String name, LongSupplier nanoTimeSupplier) { + return new UptimeMetric(name, nanoTimeSupplier); + } +} diff --git a/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java b/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java index f705089f2a..3f0890274f 100644 --- a/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java +++ b/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java @@ -22,12 +22,17 @@ import co.elastic.logstash.api.Metric; import co.elastic.logstash.api.NamespacedMetric; +import co.elastic.logstash.api.UserMetric; import org.assertj.core.data.Percentage; import org.jruby.RubyHash; import org.junit.Ignore; import org.junit.Test; import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -158,4 +163,67 @@ public void testRoot() { final NamespacedMetric namespaced = root.namespace("someothernamespace"); assertThat(namespaced.namespaceName()).containsExactly("someothernamespace"); } + + @Test + public void testRegister() { + final NamespacedMetric metrics = this.getInstance().namespace("testRegister"); + + CustomMetric leftCustomMetric = metrics.register("left", CorrelatingCustomMetric.FACTORY); + + // re-registering the same metric should get the existing instance. + assertThat(metrics.register("left", CorrelatingCustomMetric.FACTORY)).isSameAs(leftCustomMetric); + + // registering a new metric should be different instance + CustomMetric rightCustomMetric = metrics.register("right", CorrelatingCustomMetric.FACTORY); + assertThat(rightCustomMetric).isNotSameAs(leftCustomMetric); + + // this tests our test-only CustomMetric impl more than anything, but it validates + // that the instances we update are connected to their values. + leftCustomMetric.record("this"); + leftCustomMetric.record("that"); + rightCustomMetric.record("that"); + leftCustomMetric.record("this"); + rightCustomMetric.record("another"); + rightCustomMetric.record("that"); + rightCustomMetric.record("another"); + + assertThat(leftCustomMetric.getValue()).contains("this=2", "that=1"); + assertThat(rightCustomMetric.getValue()).contains("that=2", "another=2"); + } + + private interface CustomMetric extends UserMetric { + void record(final String value); + + UserMetric.Provider PROVIDER = new UserMetric.Provider(CustomMetric.class, new CustomMetric() { + @Override + public void record(String value) { + // no-op + } + + @Override + public String getValue() { + return ""; + } + }); + } + + private static class CorrelatingCustomMetric implements CustomMetric { + private final ConcurrentHashMap mapping = new ConcurrentHashMap<>(); + + static UserMetric.Factory FACTORY = CustomMetric.PROVIDER.getFactory((name) -> new CorrelatingCustomMetric()); + + @Override + public void record(String value) { + mapping.compute(value, (k, v) -> v == null ? 1 : v + 1); + } + + @Override + public String getValue() { + return Map.copyOf(mapping).entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .map((e) -> (e.getKey() + '=' + e.getValue().toString())) + .reduce((a, b) -> a + ';' + b).orElse("EMPTY"); + } + } } diff --git a/logstash-core/src/test/java/org/logstash/settings/PasswordSettingTest.java b/logstash-core/src/test/java/org/logstash/settings/PasswordSettingTest.java new file mode 100644 index 0000000000..5f75777f93 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/settings/PasswordSettingTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.logstash.settings; + +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.*; + +public class PasswordSettingTest { + + private final String SETTING_NAME = "setting_name"; + private PasswordSetting sut; + + @Before + public void setUp() { + sut = new PasswordSetting(SETTING_NAME, null, true); + } + + @Test + public void givenUnsetPasswordSetting_thenIsConsideredAsValid() { + assertNotThrown(() -> sut.validateValue()); + assertThat(sut.value(), is(instanceOf(co.elastic.logstash.api.Password.class))); + assertNull(((co.elastic.logstash.api.Password) sut.value()).getValue()); + } + + @Test + public void givenUnsetPasswordSetting_whenIsSetIsInvoked_thenReturnFalse() { + assertFalse(sut.isSet()); + } + + @Test + public void givenSetPasswordSetting_thenIsValid() { + sut.set("s3cUr3p4$$w0rd"); + + assertNotThrown(() -> sut.validateValue()); + assertThat(sut.value(), is(instanceOf(co.elastic.logstash.api.Password.class))); + assertEquals("s3cUr3p4$$w0rd", ((co.elastic.logstash.api.Password) sut.value()).getValue()); + } + + @Test + public void givenSetPasswordSetting_whenIsSetIsInvoked_thenReturnTrue() { + sut.set("s3cUr3p4$$w0rd"); + + assertTrue(sut.isSet()); + } + + @Test + public void givenSetPasswordSettingWithInvalidNonStringValue_thenRejectsTheInvalidValue() { + Exception e = assertThrows(IllegalArgumentException.class, () -> sut.set(867_5309)); + assertThat(e.getMessage(), is("Setting `" + SETTING_NAME + "` could not coerce non-string value to password")); + } + + private void assertNotThrown(Runnable test) { + try { + test.run(); + } catch (Exception e) { + fail("Exception should not be thrown"); + } + } + +} \ No newline at end of file diff --git a/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.md b/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.md new file mode 100644 index 0000000000..6daecabb14 --- /dev/null +++ b/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.md @@ -0,0 +1,141 @@ +# Summary + +The logstash data directory contains a queue for pipeline `main` containing: + + - ACK'd events (from a page that is not fully-ack'd) + - raw CBOR-encoded events + - zstd-compressed events with different compression goals + +# Pages +~~~ +1 258 821AACAC page.0 CBOR(stringref) +2 343 3BE717E8 page.0 CBOR(stringref) +3 332 3439807A page.0 CBOR(stringref) +4 258 C04209D4 page.0 CBOR(stringref) +5 343 3DFB08E8 page.0 CBOR(stringref) +6 332 44B0315D page.0 CBOR(stringref) +7 258 90D25985 page.0 CBOR(stringref) +8 343 DAFD5712 page.0 CBOR(stringref) +9 332 AB6A81DF page.0 CBOR(stringref) +10 258 157EA7A6 page.0 CBOR(stringref) +11 258 02C0F7A2 page.0 CBOR(stringref) +12 343 0005E8A8 page.0 CBOR(stringref) +13 332 C2DA39EA page.1 CBOR(stringref) +14 258 377D623C page.1 CBOR(stringref) +15 343 9F76657C page.1 CBOR(stringref) +16 332 50B51A98 page.1 CBOR(stringref) +17 258 827848CC page.1 CBOR(stringref) +18 343 8325D121 page.1 CBOR(stringref) +19 332 E1A1378B page.1 CBOR(stringref) +20 258 1BBDAA1A page.1 CBOR(stringref) +21 254 19C85DF6 page.1 ZSTD(258) +22 317 AD5DC7CC page.1 ZSTD(343) +23 325 BB8CE48C page.1 ZSTD(332) +24 254 27D38856 page.1 ZSTD(258) +25 317 67A7D2F3 page.2 ZSTD(343) +26 325 888AF9B2 page.2 ZSTD(332) +27 254 CAA2FDE3 page.2 ZSTD(258) +28 317 2985771A page.2 ZSTD(343) +29 325 89197F51 page.2 ZSTD(332) +30 254 A9E292EE page.2 ZSTD(258) +31 258 243FC2C1 page.2 CBOR(stringref) +32 219 2E2E0BDF page.2 ZSTD(258) +33 261 5ED17F40 page.2 ZSTD(343) +34 280 86BA1E80 page.2 ZSTD(332) +35 218 6A7B8C41 page.2 ZSTD(258) +36 262 08E69C4C page.2 ZSTD(343) +37 277 CD32DEBD page.2 ZSTD(332) +38 218 43101D61 page.2 ZSTD(258) +39 261 A22033DE page.3 ZSTD(343) +40 279 8F1FE0FA page.3 ZSTD(332) +41 218 FF56D05C page.3 ZSTD(258) +42 258 7077981D page.3 CBOR(stringref) +43 343 7748A127 page.3 CBOR(stringref) +44 332 B4A0C82C page.3 CBOR(stringref) +45 258 96FB0308 page.3 CBOR(stringref) +46 343 40B77975 page.3 CBOR(stringref) +47 332 D5571FDC page.3 CBOR(stringref) +48 258 BF3FC517 page.3 CBOR(stringref) +49 343 1BC62146 page.3 CBOR(stringref) +50 332 418FD829 page.3 CBOR(stringref) +51 258 DB40747E page.3 CBOR(stringref) +52 224 7629AF30 page.4 ZSTD(258) +53 264 D450FC21 page.4 ZSTD(343) +54 284 43F91F18 page.4 ZSTD(332) +55 224 C61DB7BA page.4 ZSTD(258) +56 264 F9547DBC page.4 ZSTD(343) +57 281 3DBB71E5 page.4 ZSTD(332) +58 225 8ACDB484 page.4 ZSTD(258) +59 264 8256E2D2 page.4 ZSTD(343) +60 281 D76156A2 page.4 ZSTD(332) +61 225 EDC6147B page.4 ZSTD(258) +62 258 D3AB1EF4 page.4 CBOR(stringref) +63 220 4851D677 page.4 ZSTD(258) +64 225 C8DCE54A page.4 ZSTD(258) +65 251 3D1E0F5F page.4 ZSTD(258) +66 258 1C5637CB page.4 CBOR(stringref) +67 343 09AE6714 page.5 CBOR(stringref) +68 332 4A97AC77 page.5 CBOR(stringref) +69 254 D1E43C69 page.5 ZSTD(258) +70 317 B6A2361D page.5 ZSTD(343) +71 325 A44CE35F page.5 ZSTD(332) +72 225 B69C7923 page.5 ZSTD(258) +73 265 FEBC2D45 page.5 ZSTD(343) +74 286 5FA5C389 page.5 ZSTD(332) +75 221 C36048C0 page.5 ZSTD(258) +76 262 E988C90B page.5 ZSTD(343) +77 280 6C98308C page.5 ZSTD(332) +~~~ + +# CHECKPOINTS + +~~~ +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.0 +VERSION [ 0001]: 1 +PAGENUM [ 00000000]: 0 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000005]: 5 +MINSEQN [0000000000000001]: 1 +ELEMNTS [ 0000000C]: 12 +CHECKSM [ 4AFA3119] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.1 +VERSION [ 0001]: 1 +PAGENUM [ 00000001]: 1 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [000000000000000D]: 13 +MINSEQN [000000000000000D]: 13 +ELEMNTS [ 0000000C]: 12 +CHECKSM [ 70829F7B] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.2 +VERSION [ 0001]: 1 +PAGENUM [ 00000002]: 2 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000019]: 25 +MINSEQN [0000000000000019]: 25 +ELEMNTS [ 0000000E]: 14 +CHECKSM [ 4ABFB50A] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.3 +VERSION [ 0001]: 1 +PAGENUM [ 00000003]: 3 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000027]: 39 +MINSEQN [0000000000000027]: 39 +ELEMNTS [ 0000000D]: 13 +CHECKSM [ 95B393C6] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.4 +VERSION [ 0001]: 1 +PAGENUM [ 00000004]: 4 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000034]: 52 +MINSEQN [0000000000000034]: 52 +ELEMNTS [ 0000000F]: 15 +CHECKSM [ 9B602904] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.head +VERSION [ 0001]: 1 +PAGENUM [ 00000005]: 5 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000043]: 67 +MINSEQN [0000000000000043]: 67 +ELEMNTS [ 0000000B]: 11 +CHECKSM [ B5F33B10] +~~~ \ No newline at end of file diff --git a/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.tar.gz b/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.tar.gz new file mode 100644 index 0000000000..53a0c4f985 Binary files /dev/null and b/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.tar.gz differ diff --git a/qa/integration/fixtures/pq_drain_spec.yml b/qa/integration/fixtures/pq_drain_spec.yml new file mode 100644 index 0000000000..9b135a29e7 --- /dev/null +++ b/qa/integration/fixtures/pq_drain_spec.yml @@ -0,0 +1,3 @@ +--- +services: + - logstash \ No newline at end of file diff --git a/qa/integration/services/monitoring_api.rb b/qa/integration/services/monitoring_api.rb index 34b52634f1..ceac94822a 100644 --- a/qa/integration/services/monitoring_api.rb +++ b/qa/integration/services/monitoring_api.rb @@ -74,4 +74,14 @@ def logging_reset resp = Manticore.put("http://localhost:#{@port}/_node/logging/reset", {headers: {"Content-Type" => "application/json"}}).body JSON.parse(resp) end + + def health_report + resp = Manticore.get("http://localhost:#{@port}/_health_report").body + JSON.parse(resp) + end + + def node_plugins + resp = Manticore.get("http://localhost:#{@port}/_node/plugins").body + JSON.parse(resp) + end end diff --git a/qa/integration/specs/monitoring_api_spec.rb b/qa/integration/specs/monitoring_api_spec.rb index 19dabc16f0..30598dbf92 100644 --- a/qa/integration/specs/monitoring_api_spec.rb +++ b/qa/integration/specs/monitoring_api_spec.rb @@ -205,16 +205,62 @@ end end + it 'retrieves health report' do + logstash_service = @fixture.get_service("logstash") + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # health_report can fail if the subsystem isn't ready + result = logstash_service.monitoring_api.health_report rescue nil + expect(result).not_to be_nil + expect(result).to be_a(Hash) + expect(result).to include("status") + expect(result["status"]).to match(/^(green|yellow|red)$/) + end + end + + it 'retrieves node plugins information' do + logstash_service = @fixture.get_service("logstash") + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_plugins can fail if the subsystem isn't ready + result = logstash_service.monitoring_api.node_plugins rescue nil + expect(result).not_to be_nil + expect(result).to be_a(Hash) + expect(result).to include("plugins") + plugins = result["plugins"] + expect(plugins).to be_a(Array) + expect(plugins.size).to be > 0 + # verify plugin structure and that stdin plugin is present + stdin_plugin = plugins.find { |p| p["name"] == "logstash-input-stdin" } + expect(stdin_plugin).not_to be_nil + expect(stdin_plugin).to include("name") + expect(stdin_plugin["name"]).to eq("logstash-input-stdin") + expect(stdin_plugin).to include("version") + end + end + shared_examples "pipeline metrics" do # let(:pipeline_id) { defined?(super()) or fail NotImplementedError } let(:settings_overrides) do - super().merge({'pipeline.id' => pipeline_id}) + super().dup.tap do |overrides| + overrides['pipeline.id'] = pipeline_id + if logstash_service.settings.feature_flag == "persistent_queues" + overrides['queue.compression'] = %w(none speed balanced size).sample + end + end end it "can retrieve queue stats" do logstash_service.start_with_stdin logstash_service.wait_for_logstash + number_of_events.times { + logstash_service.write_to_stdin("Testing flow metrics") + sleep(1) + } + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do # node_stats can fail if the stats subsystem isn't ready result = logstash_service.monitoring_api.node_stats rescue nil @@ -234,12 +280,90 @@ expect(queue_capacity_stats["page_capacity_in_bytes"]).not_to be_nil expect(queue_capacity_stats["max_queue_size_in_bytes"]).not_to be_nil expect(queue_capacity_stats["max_unread_events"]).not_to be_nil + queue_compression_stats = queue_stats.fetch("compression") + expect(queue_compression_stats.dig('decode', 'ratio', 'lifetime')).to be >= 1 + expect(queue_compression_stats.dig('decode', 'spend', 'lifetime')).not_to be_nil + if settings_overrides['queue.compression'] != 'none' + expect(queue_compression_stats.dig('encode', 'goal')).to eq(settings_overrides['queue.compression']) + expect(queue_compression_stats.dig('encode', 'ratio', 'lifetime')).to be <= 1 + expect(queue_compression_stats.dig('encode', 'spend', 'lifetime')).not_to be_nil + end else expect(queue_stats["type"]).to eq("memory") end end end + context "when pipeline.batch.metrics.sampling_mode is set to 'full'" do + let(:settings_overrides) do + super().merge({'pipeline.batch.metrics.sampling_mode' => 'full'}) + end + + it "can retrieve batch stats" do + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + + number_of_events.times { + logstash_service.write_to_stdin("Testing flow metrics") + sleep(1) + } + + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_stats can fail if the stats subsystem isn't ready + result = logstash_service.monitoring_api.node_stats rescue nil + expect(result).not_to be_nil + # we use fetch here since we want failed fetches to raise an exception + # and trigger the retry block + batch_stats = result.fetch("pipelines").fetch(pipeline_id).fetch("batch") + expect(batch_stats).not_to be_nil + + expect(batch_stats["event_count"]).not_to be_nil + expect(batch_stats["event_count"]["average"]).not_to be_nil + expect(batch_stats["event_count"]["average"]["lifetime"]).not_to be_nil + expect(batch_stats["event_count"]["average"]["lifetime"]).to be_a_kind_of(Numeric) + expect(batch_stats["event_count"]["average"]["lifetime"]).to be > 0 + + expect(batch_stats["event_count"]["current"]).not_to be_nil + expect(batch_stats["event_count"]["current"]).to be >= 0 + + expect(batch_stats["byte_size"]).not_to be_nil + expect(batch_stats["byte_size"]["average"]).not_to be_nil + expect(batch_stats["byte_size"]["average"]["lifetime"]).not_to be_nil + expect(batch_stats["byte_size"]["average"]["lifetime"]).to be_a_kind_of(Numeric) + expect(batch_stats["byte_size"]["average"]["lifetime"]).to be > 0 + + expect(batch_stats["byte_size"]["current"]).not_to be_nil + expect(batch_stats["byte_size"]["current"]).to be >= 0 + end + end + end + + context "when pipeline.batch.metrics.sampling_mode is set to 'disabled'" do + let(:settings_overrides) do + super().merge({'pipeline.batch.metrics.sampling_mode' => 'disabled'}) + end + + it "no batch stats metrics are available" do + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + + number_of_events.times { + logstash_service.write_to_stdin("Testing flow metrics") + sleep(1) + } + + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_stats can fail if the stats subsystem isn't ready + result = logstash_service.monitoring_api.node_stats rescue nil + expect(result).not_to be_nil + # we use fetch here since we want failed fetches to raise an exception + # and trigger the retry block + pipeline_stats = result.fetch("pipelines").fetch(pipeline_id) + expect(pipeline_stats).not_to include("batch") + end + end + end + it "retrieves the pipeline flow statuses" do logstash_service = @fixture.get_service("logstash") logstash_service.start_with_stdin diff --git a/qa/integration/specs/pq_drain_spec.rb b/qa/integration/specs/pq_drain_spec.rb new file mode 100644 index 0000000000..b524d7547b --- /dev/null +++ b/qa/integration/specs/pq_drain_spec.rb @@ -0,0 +1,142 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative '../framework/fixture' +require_relative '../framework/settings' +require_relative '../services/logstash_service' +require_relative '../framework/helpers' +require "logstash/devutils/rspec/spec_helper" + +require 'stud/temporary' + +if ENV['FEATURE_FLAG'] == 'persistent_queues' + describe "Test logstash queue draining" do + before(:all) { @fixture = Fixture.new(__FILE__) } + after(:all) { @fixture&.teardown } + + let(:logstash_service) { @fixture.get_service("logstash") } + + shared_examples 'pq drain' do |queue_compression_setting| + let(:settings_flags) { super().merge('queue.drain' => true) } + + around(:each) do |example| + Stud::Temporary.directory('data') do |tempdir| + # expand the fixture tarball into our temp data dir + data_dir_tarball = File.join(__dir__, '../fixtures/data_dirs/mixed-compression-queue-data-dir.tar.gz') + `tar --directory #{Shellwords.escape(tempdir)} --strip-components 1 -xzf "#{Shellwords.escape(data_dir_tarball)}"` + + @data_dir = tempdir + example.call + end + end + + around(:each) do |example| + Stud::Temporary.file('output') do |tempfile| + @output_file = tempfile.path + example.call + end + end + + let(:pipeline) do + <<~PIPELINE + input { generator { count => 1 type => seed } } + output { file { path => "#{@output_file}" codec => json_lines } } + PIPELINE + end + + it "reads the contents of the PQ and drains" do + + unacked_queued_elements = Pathname.new(@data_dir).glob('queue/main/checkpoint*').map { |cpf| decode_checkpoint(cpf) } + .map { |cp| (cp.elements - (cp.first_unacked_seq - cp.min_sequence)) }.reduce(&:+) + + invoke_args = %W( + --log.level=debug + --path.settings=#{File.dirname(logstash_service.application_settings_file)} + --path.data=#{@data_dir} + --pipeline.workers=2 + --pipeline.batch.size=8 + --config.string=#{pipeline} + ) + invoke_args << "-Squeue.compression=#{queue_compression_setting}" unless queue_compression_setting.nil? + + status = logstash_service.run(*invoke_args) + + aggregate_failures('process output') do + expect(status.exit_code).to be_zero + expect(status.stderr_and_stdout).to include("queue.type: persisted") + expect(status.stderr_and_stdout).to include("queue.drain: true") + expect(status.stderr_and_stdout).to include("queue.compression: #{queue_compression_setting}") unless queue_compression_setting.nil? + end + + aggregate_failures('processing result') do + # count the events, make sure they're all the right shape. + expect(::File::size(@output_file)).to_not be_zero + + written_events = ::File::read(@output_file).lines.map { |line| JSON.load(line) } + expect(written_events.size).to eq(unacked_queued_elements + 1) + timestamps = written_events.map {|event| event['@timestamp'] } + expect(timestamps.uniq.size).to eq(written_events.size) + end + + aggregate_failures('resulting queue state') do + # glob the data dir and make sure things have been cleaned up. + # we should only have a head checkpoint and a single fully-acked page. + checkpoints = Pathname.new(@data_dir).glob('queue/main/checkpoint*') + expect(checkpoints.size).to eq(1) + expect(checkpoints.first.basename.to_s).to eq('checkpoint.head') + checkpoint = decode_checkpoint(checkpoints.first) + expect(checkpoint.first_unacked_page).to eq(checkpoint.page_number) + expect(checkpoint.first_unacked_seq).to eq(checkpoint.min_sequence + checkpoint.elements) + + pages = Pathname.new(@data_dir).glob('queue/main/page*') + expect(pages.size).to eq(1) + end + end + end + + context "`queue.compression` setting" do + %w(none speed balanced size).each do |explicit_compression_setting| + context "explicit `#{explicit_compression_setting}`" do + include_examples 'pq drain', explicit_compression_setting + end + end + context "default setting" do + include_examples 'pq drain', nil + end + end + end + + def decode_checkpoint(path) + bytes = path.read(encoding: 'BINARY').bytes + + bstoi = -> (bs) { bs.reduce(0) {|m,b| (m<<8)+b } } + + version = bstoi[bytes.slice(0,2)] + pagenum = bstoi[bytes.slice(2,4)] + first_unacked_page = bstoi[bytes.slice(6,4)] + first_unacked_seq = bstoi[bytes.slice(10,8)] + min_sequence = bstoi[bytes.slice(18,8)] + elements = bstoi[bytes.slice(26,4)] + + OpenStruct.new(version: version, + page_number: pagenum, + first_unacked_page: first_unacked_page, + first_unacked_seq: first_unacked_seq, + min_sequence: min_sequence, + elements: elements) + end +end \ No newline at end of file diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 87a57d262b..2739e1fc6a 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -169,7 +169,7 @@ namespace "artifact" do desc "Generate rpm, deb, tar and zip artifacts" task "all" => ["prepare", "build"] - task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi", "build_docker_observabilitySRE"] + task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"] desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" task "archives" => ["prepare", "generate_build_metadata"] do @@ -397,52 +397,24 @@ namespace "artifact" do build_dockerfile('oss') end - namespace "dockerfile_oss" do - desc "Build Oss Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_oss"] do - build_docker_from_dockerfiles('oss') - end - end - desc "Generate Dockerfile for observability-sre images" task "dockerfile_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do puts("[dockerfiles] Building observability-sre Dockerfile") build_dockerfile('observability-sre') end - namespace "dockerfile_observabilitySRE" do - desc "Build ObservabilitySrE Docker image from Dockerfile context files" - task "docker" => ["archives_docker_observabilitySRE", "dockerfile_observabilitySRE"] do - build_docker_from_dockerfiles('observability-sre') - end - end - desc "Generate Dockerfile for full images" task "dockerfile_full" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building full Dockerfiles") build_dockerfile('full') end - namespace "dockerfile_full" do - desc "Build Full Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_full"] do - build_docker_from_dockerfiles('full') - end - end - desc "Generate Dockerfile for wolfi images" task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building wolfi Dockerfiles") build_dockerfile('wolfi') end - namespace "dockerfile_wolfi" do - desc "Build Wolfi Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_wolfi"] do - build_docker_from_dockerfiles('wolfi') - end - end - desc "Generate build context for ironbank" task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building ironbank Dockerfiles") @@ -469,30 +441,6 @@ namespace "artifact" do Rake::Task["artifact:archives_oss"].invoke end - task "build_docker_full" => [:generate_build_metadata] do - Rake::Task["artifact:docker"].invoke - Rake::Task["artifact:dockerfile_full"].invoke - Rake::Task["artifact:dockerfile_full:docker"].invoke - end - - task "build_docker_oss" => [:generate_build_metadata] do - Rake::Task["artifact:docker_oss"].invoke - Rake::Task["artifact:dockerfile_oss"].invoke - Rake::Task["artifact:dockerfile_oss:docker"].invoke - end - - task "build_docker_observabilitySRE" => [:generate_build_metadata] do - Rake::Task["artifact:docker_observabilitySRE"].invoke - Rake::Task["artifact:dockerfile_observabilitySRE"].invoke - Rake::Task["artifact:dockerfile_observabilitySRE:docker"].invoke - end - - task "build_docker_wolfi" => [:generate_build_metadata] do - Rake::Task["artifact:docker_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi:docker"].invoke - end - task "generate_build_metadata" do require 'time' require 'tempfile' @@ -927,27 +875,13 @@ namespace "artifact" do "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), "RELEASE" => ENV["RELEASE"], "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS + "BUILD_DATE" => BUILD_DATE } Dir.chdir("docker") do |dir| safe_system(env, "make build-from-local-#{flavor}-artifacts") end end - def build_docker_from_dockerfiles(flavor) - env = { - "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), - "RELEASE" => ENV["RELEASE"], - "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS - } - Dir.chdir("docker") do |dir| - safe_system(env, "make build-from-dockerfiles_#{flavor}") - end - end - def build_dockerfile(flavor) env = { "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index d95e9fcc17..0000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,12 +0,0 @@ -sonar.projectKey=elastic_logstash_AYm_nEbQaV3I-igkX1q9 -sonar.host.url=https://sonar.elastic.dev - -sonar.exclusions=vendor/**, gradle/**, rakelib/**, logstash-core-plugin-api/**, licenses/**, qa/**, **/spec/** -sonar.tests=logstash-core/src/test, x-pack/src/test, buildSrc/src/test - -# Ruby -sonar.ruby.coverage.reportPaths=coverage/coverage.json - -# Java -sonar.coverage.jacoco.xmlReportPaths=**/jacocoTestReport.xml -sonar.java.binaries=**/build/classes \ No newline at end of file diff --git a/tools/dependencies-report/src/main/resources/licenseMapping.csv b/tools/dependencies-report/src/main/resources/licenseMapping.csv index 128eef5996..14fb7c8c28 100644 --- a/tools/dependencies-report/src/main/resources/licenseMapping.csv +++ b/tools/dependencies-report/src/main/resources/licenseMapping.csv @@ -33,6 +33,7 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL "com.fasterxml.jackson.core:jackson-databind:",https://github.com/FasterXML/jackson-databind,Apache-2.0 "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:",https://github.com/FasterXML/jackson-dataformats-binary,Apache-2.0 "com.fasterxml.jackson.module:jackson-module-afterburner:",https://github.com/FasterXML/jackson-modules-base,Apache-2.0 +"com.github.luben:zstd-jni:1.5.7-4",https://github.com/luben/zstd-jni,BSD-2-Clause "com.google.googlejavaformat:google-java-format:",https://github.com/google/google-java-format,Apache-2.0 "com.google.guava:guava:",https://github.com/google/guava,Apache-2.0 "com.google.j2objc:j2objc-annotations:",https://github.com/google/j2objc/,Apache-2.0 diff --git a/tools/dependencies-report/src/main/resources/notices/com.github.luben!zstd-jni-NOTICE.txt b/tools/dependencies-report/src/main/resources/notices/com.github.luben!zstd-jni-NOTICE.txt new file mode 100644 index 0000000000..4accd5fd41 --- /dev/null +++ b/tools/dependencies-report/src/main/resources/notices/com.github.luben!zstd-jni-NOTICE.txt @@ -0,0 +1,26 @@ +source: https://github.com/luben/zstd-jni/blob/v1.5.7-4/LICENSE + +Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. + +BSD-2-Clause License https://opensource.org/license/bsd-2-clause + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java index 24edfec733..bb00a81c6b 100644 --- a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java +++ b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java @@ -53,7 +53,9 @@ public class JvmOptionsParser { private static final String[] MANDATORY_JVM_OPTIONS = new String[]{ "-Djruby.regexp.interruptible=true", + "-Djruby.compile.invokedynamic=true", "-Djdk.io.File.enableADS=true", + "-Dlog4j2.isThreadContextMapInheritable=true", "16-:--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", "16-:--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", "16-:--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", diff --git a/tools/jvm-options-parser/src/test/java/org/logstash/launchers/JvmOptionsParserTest.java b/tools/jvm-options-parser/src/test/java/org/logstash/launchers/JvmOptionsParserTest.java index 49e28df7f8..b61414ca89 100644 --- a/tools/jvm-options-parser/src/test/java/org/logstash/launchers/JvmOptionsParserTest.java +++ b/tools/jvm-options-parser/src/test/java/org/logstash/launchers/JvmOptionsParserTest.java @@ -122,6 +122,10 @@ public void testAlwaysMandatoryJvmPresent() { JvmOptionsParser.getMandatoryJvmOptions(11).contains("-Djruby.regexp.interruptible=true")); assertTrue("Contains regexp interruptible for Java 17", JvmOptionsParser.getMandatoryJvmOptions(17).contains("-Djruby.regexp.interruptible=true")); + assertTrue("Contains compile invokedynamic for Java 17", + JvmOptionsParser.getMandatoryJvmOptions(17).contains("-Djruby.compile.invokedynamic=true")); + assertTrue("Contains log4j2 isThreadContextMapInheritable for Java 17", + JvmOptionsParser.getMandatoryJvmOptions(17).contains("-Dlog4j2.isThreadContextMapInheritable=true")); } diff --git a/tools/release/generate_release_notes_md.rb b/tools/release/generate_release_notes_md.rb index 36baeb3f3a..d2c16c339f 100755 --- a/tools/release/generate_release_notes_md.rb +++ b/tools/release/generate_release_notes_md.rb @@ -42,7 +42,7 @@ coming_tag_index = release_notes.find_index {|line| line.match(/^## #{current_release} \[logstash-#{current_release}-release-notes\]$/) } coming_tag_index += 1 if coming_tag_index -release_notes_entry_index = coming_tag_index || release_notes.find_index {|line| line.match(/\[logstash-\d+-release-notes\]$/) } +release_notes_entry_index = coming_tag_index || release_notes.find_index {|line| line.match(/^## .*\[logstash-.*-release-notes\]$/) } unless coming_tag_index report << "## #{current_release} [logstash-#{current_release}-release-notes]\n\n" @@ -91,15 +91,17 @@ plugin_changes.each do |plugin, versions| _, type, name = plugin.split("-") header = "**#{name.capitalize} #{type.capitalize} - #{versions.last}**" + # Determine the correct GitHub organization + org = plugin.include?('elastic_integration') ? 'elastic' : 'logstash-plugins' start_changelog_file = Tempfile.new(plugin + 'start') end_changelog_file = Tempfile.new(plugin + 'end') - changelog = `curl https://raw.githubusercontent.com/logstash-plugins/#{plugin}/v#{versions.last}/CHANGELOG.md`.split("\n") + changelog = `curl https://raw.githubusercontent.com/#{org}/#{plugin}/v#{versions.last}/CHANGELOG.md`.split("\n") report << "#{header}\n" changelog.each do |line| break if line.match(/^## #{versions.first}/) next if line.match(/^##/) line.gsub!(/^\+/, "") - line.gsub!(/ #(?\d+)\s*$/, " https://github.com/logstash-plugins/#{plugin}/issues/\\k[#\\k]") + line.gsub!(/ #(?\d+)\s*$/, " https://github.com/#{org}/#{plugin}/issues/\\k[#\\k]") line.gsub!(/\[#(?\d+)\]\((?[^)]*)\)/, "[#\\k](\\k)") line.gsub!(/^\s+-/, "*") report << line diff --git a/versions.yml b/versions.yml index bad7755a11..02422e9021 100644 --- a/versions.yml +++ b/versions.yml @@ -1,7 +1,7 @@ --- # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var -logstash: 9.2.0 -logstash-core: 9.2.0 +logstash: 9.3.0 +logstash-core: 9.3.0 logstash-core-plugin-api: 2.1.16 bundled_jdk: diff --git a/x-pack/lib/config_management/bootstrap_check.rb b/x-pack/lib/config_management/bootstrap_check.rb index 89b675d0bb..06067fe45d 100644 --- a/x-pack/lib/config_management/bootstrap_check.rb +++ b/x-pack/lib/config_management/bootstrap_check.rb @@ -14,7 +14,7 @@ module ConfigManagement class BootstrapCheck include LogStash::Util::Loggable - # pipeline ID must begin with a letter or underscore and contain only letters, underscores, dashes, and numbers + # pipeline ID must begin with a letter or underscore and contain only letters, underscores, dashes, hyphens, and numbers # wildcard character `*` is also acceptable and follows globbing rules PIPELINE_ID_PATTERN = %r{\A[a-z_*][a-z_\-0-9*]*\Z}i @@ -43,7 +43,7 @@ def self.check(settings) invalid_patterns = pipeline_ids.reject { |entry| PIPELINE_ID_PATTERN =~ entry } if invalid_patterns.any? - raise LogStash::BootstrapCheckError, "Pipeline id in `xpack.management.pipeline.id` must begin with a letter or underscore and contain only letters, underscores, dashes, and numbers. The asterisk wildcard `*` can also be used. Invalid ids: #{invalid_patterns.join(', ')}" + raise LogStash::BootstrapCheckError, "Pipeline id in `xpack.management.pipeline.id` must begin with a letter or underscore and contain only letters, underscores, dashes, hyphens, and numbers. The asterisk wildcard `*` can also be used. Invalid ids: #{invalid_patterns.join(', ')}" end duplicate_ids = find_duplicate_ids(pipeline_ids) diff --git a/x-pack/spec/config_management/elasticsearch_source_spec.rb b/x-pack/spec/config_management/elasticsearch_source_spec.rb index b16916ca5e..6d51df9224 100644 --- a/x-pack/spec/config_management/elasticsearch_source_spec.rb +++ b/x-pack/spec/config_management/elasticsearch_source_spec.rb @@ -170,6 +170,30 @@ expect { described_class.new(system_settings) }.to_not raise_error end end + + context "when api_key is set (encoded or not)" do + [ + { desc: "non-encoded", value: "foo:bar" }, + { desc: "encoded", value: Base64.strict_encode64("foo:bar") } + ].each do |api_key_case| + context "with #{api_key_case[:desc]} api_key" do + let(:settings) do + { + "xpack.management.enabled" => true, + "xpack.management.pipeline.id" => "main", + "xpack.management.elasticsearch.api_key" => api_key_case[:value], + } + end + + it "will rely on #{api_key_case[:desc]} api_key for authentication" do + # the http client used by xpack module is the same as the one used by the ES output plugin + # and the HttpClientBuilder.setup_api_key method will handle both encoded and non-encoded api_key values. + # These tests prevent future regressions if the plugin client is changed. + expect { described_class.new(system_settings) }.to_not raise_error + end + end + end + end end context "valid settings" do