From 034045e7268deb012445400c4baf240a8d19bf7b Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 27 Apr 2026 15:40:58 -0700 Subject: [PATCH 1/6] Upgrade elasticsearch ruby client and remove 7.x stack related patch logics. --- lib/logstash/filters/elasticsearch.rb | 3 - .../filters/elasticsearch/dsl_executor.rb | 7 +- ..._elasticsearch_transport_http_manticore.rb | 44 ------------ logstash-filter-elasticsearch.gemspec | 4 +- spec/filters/elasticsearch_dsl_spec.rb | 20 ------ spec/filters/elasticsearch_spec.rb | 19 ++--- spec/filters/elasticsearch_ssl_spec.rb | 2 +- ...lasticsearch_7.x_hits_total_as_object.json | 70 ------------------- 8 files changed, 8 insertions(+), 161 deletions(-) delete mode 100644 lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb delete mode 100644 spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json diff --git a/lib/logstash/filters/elasticsearch.rb b/lib/logstash/filters/elasticsearch.rb index ce133a56..9f4f9913 100644 --- a/lib/logstash/filters/elasticsearch.rb +++ b/lib/logstash/filters/elasticsearch.rb @@ -209,9 +209,6 @@ def register test_connection! validate_es_for_esql_support! if @query_type == "esql" setup_serverless - if get_client.es_transport_client_type == "elasticsearch_transport" - require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore" - end end # def register def filter(event) diff --git a/lib/logstash/filters/elasticsearch/dsl_executor.rb b/lib/logstash/filters/elasticsearch/dsl_executor.rb index bf92050b..12eeb955 100644 --- a/lib/logstash/filters/elasticsearch/dsl_executor.rb +++ b/lib/logstash/filters/elasticsearch/dsl_executor.rb @@ -98,12 +98,7 @@ def process(client, event) # @return [Integer] def extract_total_from_hits(hits) total = hits['total'] - - # Elasticsearch 7.x produces an object containing `value` and `relation` in order - # to enable unambiguous reporting when the total is only a lower bound; if we get - # an object back, return its `value`. - return total['value'] if total.kind_of?(Hash) - total + total.kind_of?(Hash) ? total['value'] : total end # get an array of path elements from a path reference diff --git a/lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb b/lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb deleted file mode 100644 index c8b1f0b0..00000000 --- a/lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb +++ /dev/null @@ -1,44 +0,0 @@ -# encoding: utf-8 -require "elasticsearch" -require "elasticsearch/transport/transport/http/manticore" - -es_client_version = Gem.loaded_specs['elasticsearch-transport'].version -if es_client_version >= Gem::Version.new('7.2') && es_client_version < Gem::Version.new('7.16') - # elasticsearch-transport 7.2.0 - 7.14.0 had a bug where setting http headers - # ES::Client.new ..., transport_options: { headers: { 'Authorization' => ... } } - # would be lost https://github.com/elastic/elasticsearch-ruby/issues/1428 - # - # NOTE: needs to be idempotent as input ES plugin might apply the same patch! - # - # @private - module Elasticsearch - module Transport - module Transport - module HTTP - class Manticore - - def apply_headers(request_options, options) - headers = (options && options[:headers]) || {} - headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE - - # this code is necessary to grab the correct user-agent header - # when this method is invoked with apply_headers(@request_options, options) - # from https://github.com/elastic/elasticsearch-ruby/blob/v7.14.0/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb#L113-L114 - transport_user_agent = nil - if (options && options[:transport_options] && options[:transport_options][:headers]) - transport_headers = {} - transport_headers = options[:transport_options][:headers] - transport_user_agent = find_value(transport_headers, USER_AGENT_REGEX) - end - - headers[USER_AGENT_STR] = transport_user_agent || find_value(headers, USER_AGENT_REGEX) || user_agent_header - headers[ACCEPT_ENCODING] = GZIP if use_compression? - (request_options[:headers] ||= {}).merge!(headers) # this line was changed - end - - end - end - end - end - end -end \ No newline at end of file diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec index 55c24b88..c3c0eb73 100644 --- a/logstash-filter-elasticsearch.gemspec +++ b/logstash-filter-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-elasticsearch' - s.version = '4.3.1' + s.version = '4.3.2' s.licenses = ['Apache License (2.0)'] s.summary = "Copies fields from previous log events in Elasticsearch to current events " s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -21,7 +21,7 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - s.add_runtime_dependency 'elasticsearch', ">= 7.14.9", '< 9' + s.add_runtime_dependency 'elasticsearch', '>= 8', '< 10' s.add_runtime_dependency 'manticore', ">= 0.7.1" s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3' s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0' diff --git a/spec/filters/elasticsearch_dsl_spec.rb b/spec/filters/elasticsearch_dsl_spec.rb index 2885caa6..df56e488 100644 --- a/spec/filters/elasticsearch_dsl_spec.rb +++ b/spec/filters/elasticsearch_dsl_spec.rb @@ -123,26 +123,6 @@ end end - context 'when Elasticsearch 7.x gives us a totals object instead of an integer' do - let(:plugin_config) do - { - "hosts" => ["localhost:9200"], - "query" => "response: 404", - "fields" => { "response" => "code" }, - "result_size" => 10 - } - end - - let(:response) do - LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "elasticsearch_7.x_hits_total_as_object.json"))) - end - - it "should enhance the current event with new data" do - plugin.filter(event) - expect(event.get("[@metadata][total_hits]")).to eq(13476) - end - end - context "if something wrong happen during connection" do before(:each) do diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb index b94da52e..1b7ed449 100644 --- a/spec/filters/elasticsearch_spec.rb +++ b/spec/filters/elasticsearch_spec.rb @@ -119,15 +119,15 @@ def initialize() "cluster_name": "docker-cluster", "cluster_uuid": "DyR1hN03QvuCWXRy3jtb0g", "version": { - "number": "7.13.1", + "number": "8.0.0", "build_flavor": "default", "build_type": "docker", "build_hash": "9a7758028e4ea59bcab41c12004603c5a7dd84a9", "build_date": "2021-05-28T17:40:59.346932922Z", "build_snapshot": false, - "lucene_version": "8.8.2", - "minimum_wire_compatibility_version": "6.8.0", - "minimum_index_compatibility_version": "6.0.0-beta1" + "lucene_version": "9.0.0", + "minimum_wire_compatibility_version": "7.17.0", + "minimum_index_compatibility_version": "7.0.0" }, "tagline": "You Know, for Search" } @@ -188,17 +188,6 @@ def wait_receive_request end let(:plugin) { described_class.new(config) } let(:event) { LogStash::Event.new({}) } - - # elasticsearch-ruby 7.17.9 initialize two user agent headers, `user-agent` and `User-Agent` - # hence, fail this header size test case - xit "client should sent the expect user-agent" do - plugin.register - - request = webserver.wait_receive_request - - expect(request.header['user-agent'].size).to eq(1) - expect(request.header['user-agent'][0]).to match(/logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-filter-elasticsearch\/\d*\.\d*\.\d*/) - end end end diff --git a/spec/filters/elasticsearch_ssl_spec.rb b/spec/filters/elasticsearch_ssl_spec.rb index 1cdc80fb..e1f5c81b 100644 --- a/spec/filters/elasticsearch_ssl_spec.rb +++ b/spec/filters/elasticsearch_ssl_spec.rb @@ -15,7 +15,7 @@ before do allow(es_client_double).to receive(:close) allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object) - allow(es_client_double).to receive(:info).with(any_args).and_return({"version" => {"number" => "7.5.0", "build_flavor" => "default"}, + allow(es_client_double).to receive(:info).with(any_args).and_return({"version" => {"number" => "8.0.0", "build_flavor" => "default"}, "tagline" => "You Know, for Search"}) allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double) end diff --git a/spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json b/spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json deleted file mode 100644 index 79f9a1bf..00000000 --- a/spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "took": 49, - "timed_out": false, - "_shards": { - "total": 155, - "successful": 155, - "failed": 0 - }, - "hits": { - "total": { - "value": 13476, - "relation": "eq" - }, - "max_score": 1, - "hits": [{ - "_index": "logstash-2014.08.26", - "_type": "logs", - "_id": "AVVY76L_AW7v0kX8KXo4", - "_score": 1, - "_source": { - "request": "/doc/index.html?org/elasticsearch/action/search/SearchResponse.html", - "agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"", - "geoip": { - "timezone": "America/Los_Angeles", - "ip": "66.249.73.185", - "latitude": 37.386, - "continent_code": "NA", - "city_name": "Mountain View", - "country_code2": "US", - "country_name": "United States", - "dma_code": 807, - "country_code3": "US", - "region_name": "California", - "location": [-122.0838, - 37.386 - ], - "postal_code": "94035", - "longitude": -122.0838, - "region_code": "CA" - }, - "auth": "-", - "ident": "-", - "verb": "GET", - "useragent": { - "os": "Other", - "major": "2", - "minor": "1", - "name": "Googlebot", - "os_name": "Other", - "device": "Spider" - }, - "message": "66.249.73.185 - - [26/Aug/2014:21:22:13 +0000] \"GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1\" 404 294 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"", - "referrer": "\"-\"", - "@timestamp": "2014-08-26T21:22:13.000Z", - "response": 404, - "bytes": 294, - "clientip": "66.249.73.185", - "@version": "1", - "host": "skywalker", - "httpversion": "1.1", - "timestamp": "26/Aug/2014:21:22:13 +0000" - } - }] - }, - "aggregations": { - "bytes_avg": { - "value": 294 - } - } -} From dd3268aef9a7337836e99beba53058995afa6a91 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 27 Apr 2026 16:09:55 -0700 Subject: [PATCH 2/6] Remove old ES client logics from internal ES client class. --- lib/logstash/filters/elasticsearch/client.rb | 11 +---------- spec/filters/elasticsearch_dsl_spec.rb | 10 ---------- spec/filters/elasticsearch_spec.rb | 12 +----------- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index 9ac1a6f7..b40de72f 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -8,7 +8,6 @@ module Filters class ElasticsearchClient attr_reader :client - attr_reader :es_transport_client_type BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze @@ -114,16 +113,8 @@ def base64?(string) end def get_transport_client_class - # LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport` - # And now `elasticsearch-transport` is old, instead we have `elastic-transport`. - # LS-core updated `elasticsearch` > 8: https://github.com/elastic/logstash/pull/17161 - # Following source bits are for the compatibility to support both `elasticsearch-transport` and `elastic-transport` gems - require "elasticsearch/transport/transport/http/manticore" - es_transport_client_type = "elasticsearch_transport" - ::Elasticsearch::Transport::Transport::HTTP::Manticore - rescue ::LoadError + # Elasticsearch 8+ uses `elastic-transport` gem (part of the elasticsearch gem) require "elastic/transport/transport/http/manticore" - es_transport_client_type = "elastic_transport" ::Elastic::Transport::Transport::HTTP::Manticore end end diff --git a/spec/filters/elasticsearch_dsl_spec.rb b/spec/filters/elasticsearch_dsl_spec.rb index df56e488..a37d619b 100644 --- a/spec/filters/elasticsearch_dsl_spec.rb +++ b/spec/filters/elasticsearch_dsl_spec.rb @@ -58,11 +58,6 @@ before(:each) do allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client) - if defined?(Elastic::Transport) - allow(client).to receive(:es_transport_client_type).and_return('elastic_transport') - else - allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport') - end allow(client).to receive(:search).and_return(response) allow(plugin).to receive(:test_connection!) allow(plugin).to receive(:setup_serverless) @@ -287,11 +282,6 @@ before do allow(plugin).to receive(:get_client).and_return(client_double) - if defined?(Elastic::Transport) - allow(client_double).to receive(:es_transport_client_type).and_return('elastic_transport') - else - allow(client_double).to receive(:es_transport_client_type).and_return('elasticsearch_transport') - end allow(client_double).to receive(:client).and_return(transport_double) end diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb index 1b7ed449..c37c9e8f 100644 --- a/spec/filters/elasticsearch_spec.rb +++ b/spec/filters/elasticsearch_spec.rb @@ -215,12 +215,7 @@ def wait_receive_request # this spec is a safeguard to trigger an assessment of thread-safety should # we choose a different transport adapter in the future. transport_class = extract_transport(client).options.fetch(:transport_class) - if defined?(Elastic::Transport) - allow(client).to receive(:es_transport_client_type).and_return("elastic_transport") - expect(transport_class).to equal ::Elastic::Transport::Transport::HTTP::Manticore - else - expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore - end + expect(transport_class).to equal ::Elastic::Transport::Transport::HTTP::Manticore end it 'uses a client with sufficient connection pool size' do @@ -549,11 +544,6 @@ def wait_receive_request before(:each) do allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client) - if defined?(Elastic::Transport) - allow(client).to receive(:es_transport_client_type).and_return('elastic_transport') - else - allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport') - end allow(plugin).to receive(:test_connection!) allow(plugin).to receive(:setup_serverless) plugin.register From e56e061208cf16c79a83f8b9f3d817d06761b99c Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 4 May 2026 12:09:19 -0700 Subject: [PATCH 3/6] Simplify client logic to directly utilize manticore transport client as it was before depending on the client type. Remove dead spec logics never get executed. --- lib/logstash/filters/elasticsearch/client.rb | 12 +++--------- spec/filters/elasticsearch_spec.rb | 14 +++----------- spec/filters/integration/elasticsearch_spec.rb | 10 +--------- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index b40de72f..5442f0b9 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -1,7 +1,7 @@ # encoding: utf-8 -require "elasticsearch" require "base64" - +require "elasticsearch" +require "elastic/transport/transport/http/manticore" module LogStash module Filters @@ -43,7 +43,7 @@ def initialize(logger, hosts, options = {}) client_options = { hosts: hosts, - transport_class: get_transport_client_class, + transport_class: ::Elastic::Transport::Transport::HTTP::Manticore, transport_options: transport_options, ssl: ssl_options, retry_on_failure: options[:retry_on_failure], @@ -111,12 +111,6 @@ def base64?(string) rescue ArgumentError false end - - def get_transport_client_class - # Elasticsearch 8+ uses `elastic-transport` gem (part of the elasticsearch gem) - require "elastic/transport/transport/http/manticore" - ::Elastic::Transport::Transport::HTTP::Manticore - end end end end diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb index c37c9e8f..9a3a6bae 100644 --- a/spec/filters/elasticsearch_spec.rb +++ b/spec/filters/elasticsearch_spec.rb @@ -62,17 +62,9 @@ allow(filter_client).to receive(:serverless?).and_return(true) allow(filter_client).to receive(:client).and_return(es_client) - if defined?(Elastic::Transport) - allow(es_client).to receive(:info) - .with(a_hash_including( - :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)) - .and_raise(Elastic::Transport::Transport::Errors::BadRequest.new) - else - allow(es_client).to receive(:info) - .with(a_hash_including( - :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)) - .and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest.new) - end + allow(es_client).to receive(:info) + .with(a_hash_including(:headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)) + .and_raise(Elastic::Transport::Transport::Errors::BadRequest.new) end it "raises an exception when Elastic Api Version is not supported" do diff --git a/spec/filters/integration/elasticsearch_spec.rb b/spec/filters/integration/elasticsearch_spec.rb index 8895e8d4..822429b5 100644 --- a/spec/filters/integration/elasticsearch_spec.rb +++ b/spec/filters/integration/elasticsearch_spec.rb @@ -84,9 +84,7 @@ end it "fails to register plugin" do - expect { plugin.register }.to raise_error elastic_ruby_v8_client_available? ? - Elastic::Transport::Transport::Errors::Unauthorized : - Elasticsearch::Transport::Transport::Errors::Unauthorized + expect { plugin.register }.to raise_error Elastic::Transport::Transport::Errors::Unauthorized end end if ELASTIC_SECURITY_ENABLED @@ -152,10 +150,4 @@ end end end - def elastic_ruby_v8_client_available? - Elasticsearch::Transport - false - rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available - true - end end From b5b0817a1146b5b475c1e39d0e4abcac5a3e9418 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 4 May 2026 13:30:37 -0700 Subject: [PATCH 4/6] Put back the ES 7.x comment --- lib/logstash/filters/elasticsearch/dsl_executor.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/logstash/filters/elasticsearch/dsl_executor.rb b/lib/logstash/filters/elasticsearch/dsl_executor.rb index 12eeb955..bf92050b 100644 --- a/lib/logstash/filters/elasticsearch/dsl_executor.rb +++ b/lib/logstash/filters/elasticsearch/dsl_executor.rb @@ -98,7 +98,12 @@ def process(client, event) # @return [Integer] def extract_total_from_hits(hits) total = hits['total'] - total.kind_of?(Hash) ? total['value'] : total + + # Elasticsearch 7.x produces an object containing `value` and `relation` in order + # to enable unambiguous reporting when the total is only a lower bound; if we get + # an object back, return its `value`. + return total['value'] if total.kind_of?(Hash) + total end # get an array of path elements from a path reference From 6dfb9a5dcc4709c1e3b168746f7dc58c9a64eee5 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 4 May 2026 13:41:05 -0700 Subject: [PATCH 5/6] Add changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d2eeb3a..0c4d16ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.3.2 + - Remove elasticsearch-ruby 7.x patch logics and allow to update it to 9.x version [#213](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/213) + ## 4.3.1 - Added support for encoded and non encoded api-key formats on plugin configuration [#203](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/203) From a59a92d83e9c25f81846d1f7c7d160e852899ca1 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Wed, 20 May 2026 08:26:40 -0700 Subject: [PATCH 6/6] Remove changelog to leverage version-bump/add-changelog workflow automation --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c4d16ff..4d2eeb3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,3 @@ -## 4.3.2 - - Remove elasticsearch-ruby 7.x patch logics and allow to update it to 9.x version [#213](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/213) - ## 4.3.1 - Added support for encoded and non encoded api-key formats on plugin configuration [#203](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/203)