From 635424a1c74624676d43bdd2465e7ae6a5b45430 Mon Sep 17 00:00:00 2001 From: Neeme Praks Date: Mon, 9 Feb 2026 18:39:14 +0200 Subject: [PATCH] Add configurable connection pool settings for OTLP exporter Add CLI arguments to configure HTTP connection pool behavior: - --otlp-exporter-pool-idle-timeout (default: 30s) - --otlp-exporter-pool-max-idle-per-host (default: 100) These settings allow tuning connection reuse for different deployment scenarios, such as long-lived connections to collectors. Addresses the TODO in src/exporters/http/client.rs. --- README.md | 2 ++ src/exporters/awsemf/cloudwatch.rs | 10 +++++++++- src/exporters/awsemf/mod.rs | 11 ++++++++++- src/exporters/clickhouse/mod.rs | 11 ++++++++++- src/exporters/datadog/mod.rs | 11 ++++++++++- src/exporters/http/client.rs | 26 +++++++++++++++++++++----- src/exporters/otlp/client.rs | 19 +++++++++++++++++-- src/exporters/otlp/config.rs | 15 +++++++++++++++ src/exporters/otlp/exporter.rs | 6 ++++++ src/exporters/xray/mod.rs | 11 ++++++++++- src/init/otlp_exporter.rs | 25 ++++++++++++++++++++++++- 11 files changed, 134 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9fec46aa..98230bf1 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,8 @@ The OTLP exporter is the default, or can be explicitly selected with `--exporter | --otlp-exporter-tls-ca-pem | | | | --otlp-exporter-tls-skip-verify | | | | --otlp-exporter-request-timeout | 5s | | +| --otlp-exporter-pool-idle-timeout | 30s | | +| --otlp-exporter-pool-max-idle-per-host | 100 | | | --otlp-exporter-retry-initial-backoff | (uses global exporter default) | | | --otlp-exporter-retry-max-backoff | (uses global exporter default) | | | --otlp-exporter-retry-max-elapsed-time | (uses global exporter default) | | diff --git a/src/exporters/awsemf/cloudwatch.rs b/src/exporters/awsemf/cloudwatch.rs index 3c4bdf0f..332c106d 100644 --- a/src/exporters/awsemf/cloudwatch.rs +++ b/src/exporters/awsemf/cloudwatch.rs @@ -51,8 +51,16 @@ impl Cloudwatch { HeaderValue::from_static("application/x-amz-json-1.1"), ); + use crate::exporters::http::client::{ + DEFAULT_POOL_IDLE_TIMEOUT, DEFAULT_POOL_MAX_IDLE_PER_HOST, + }; // Use the existing HTTP client builder - let client = build_hyper_client(Config::default(), false)?; + let client = build_hyper_client( + Config::default(), + false, + DEFAULT_POOL_IDLE_TIMEOUT, + DEFAULT_POOL_MAX_IDLE_PER_HOST, + )?; Ok(Self { endpoint, diff --git a/src/exporters/awsemf/mod.rs b/src/exporters/awsemf/mod.rs index ce4aa88e..d433ac90 100644 --- a/src/exporters/awsemf/mod.rs +++ b/src/exporters/awsemf/mod.rs @@ -174,7 +174,16 @@ impl AwsEmfExporterBuilder { flush_receiver: Option, aws_creds_provider: AwsCredsProvider, ) -> Result, BoxError> { - let client = Client::build(tls::Config::default(), Protocol::Http, Default::default())?; + use crate::exporters::http::client::{ + DEFAULT_POOL_IDLE_TIMEOUT, DEFAULT_POOL_MAX_IDLE_PER_HOST, + }; + let client = Client::build( + tls::Config::default(), + Protocol::Http, + Default::default(), + DEFAULT_POOL_IDLE_TIMEOUT, + DEFAULT_POOL_MAX_IDLE_PER_HOST, + )?; let dim_filter = Arc::new(DimensionFilter::new( self.config.include_dimensions.clone(), self.config.exclude_dimensions.clone(), diff --git a/src/exporters/clickhouse/mod.rs b/src/exporters/clickhouse/mod.rs index e1da36a4..cd4ee786 100644 --- a/src/exporters/clickhouse/mod.rs +++ b/src/exporters/clickhouse/mod.rs @@ -219,7 +219,16 @@ impl ClickhouseExporterBuilder { Resource: Clone + Send + Sync + 'static, Transformer: TransformPayload, { - let client = Client::build(tls::Config::default(), Protocol::Http, Default::default())?; + use crate::exporters::http::client::{ + DEFAULT_POOL_IDLE_TIMEOUT, DEFAULT_POOL_MAX_IDLE_PER_HOST, + }; + let client = Client::build( + tls::Config::default(), + Protocol::Http, + Default::default(), + DEFAULT_POOL_IDLE_TIMEOUT, + DEFAULT_POOL_MAX_IDLE_PER_HOST, + )?; let transformer = Transformer::new(self.config.compression.clone(), self.config.use_json); diff --git a/src/exporters/datadog/mod.rs b/src/exporters/datadog/mod.rs index 5a22935e..291ec25d 100644 --- a/src/exporters/datadog/mod.rs +++ b/src/exporters/datadog/mod.rs @@ -148,7 +148,16 @@ impl DatadogExporterBuilder { rx: BoundedReceiver>>, flush_receiver: Option, ) -> Result, BoxError> { - let client = Client::build(tls::Config::default(), Protocol::Http, Default::default())?; + use crate::exporters::http::client::{ + DEFAULT_POOL_IDLE_TIMEOUT, DEFAULT_POOL_MAX_IDLE_PER_HOST, + }; + let client = Client::build( + tls::Config::default(), + Protocol::Http, + Default::default(), + DEFAULT_POOL_IDLE_TIMEOUT, + DEFAULT_POOL_MAX_IDLE_PER_HOST, + )?; let transformer = Transformer::new(self.environment, self.hostname); diff --git a/src/exporters/http/client.rs b/src/exporters/http/client.rs index 675b9c5a..f4f1bdc3 100644 --- a/src/exporters/http/client.rs +++ b/src/exporters/http/client.rs @@ -30,6 +30,10 @@ use tonic::Status; use tower::{BoxError, Service}; use tracing::warn; +// Default connection pool settings +pub const DEFAULT_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(30); +pub const DEFAULT_POOL_MAX_IDLE_PER_HOST: usize = 100; + #[derive(Clone, Debug, PartialEq)] pub enum Protocol { Grpc, @@ -78,6 +82,8 @@ pub trait ResponseDecode { pub(crate) fn build_hyper_client( tls_config: Config, http2_only: bool, + pool_idle_timeout: Duration, + pool_max_idle_per_host: usize, ) -> Result, ReqBody>, BoxError> where ReqBody: Body + Send, @@ -92,9 +98,8 @@ where .build(); let client = hyper_util::client::legacy::Client::builder(TokioExecutor::new()) - // todo: make configurable - .pool_idle_timeout(Duration::from_secs(30)) - .pool_max_idle_per_host(100) + .pool_idle_timeout(pool_idle_timeout) + .pool_max_idle_per_host(pool_max_idle_per_host) .http2_only(http2_only) .timer(TokioTimer::new()) .build::<_, ReqBody>(https); @@ -107,8 +112,19 @@ where ReqBody: Body + Send, ::Data: Send, { - pub fn build(tls_config: Config, protocol: Protocol, decoder: Dec) -> Result { - let inner = build_hyper_client(tls_config, protocol == Protocol::Grpc)?; + pub fn build( + tls_config: Config, + protocol: Protocol, + decoder: Dec, + pool_idle_timeout: Duration, + pool_max_idle_per_host: usize, + ) -> Result { + let inner = build_hyper_client( + tls_config, + protocol == Protocol::Grpc, + pool_idle_timeout, + pool_max_idle_per_host, + )?; Ok(Self { inner, diff --git a/src/exporters/otlp/client.rs b/src/exporters/otlp/client.rs index a46ace44..0bdf15a6 100644 --- a/src/exporters/otlp/client.rs +++ b/src/exporters/otlp/client.rs @@ -20,6 +20,7 @@ use std::future::Future; use std::marker::PhantomData; use std::pin::Pin; use std::task::{Context, Poll}; +use std::time::Duration; use tonic::codegen::Service; use tower::BoxError; @@ -189,16 +190,30 @@ where sent: RotelCounter, send_failed: RotelCounter, signing_builder: AwsSigningServiceBuilder, + pool_idle_timeout: Duration, + pool_max_idle_per_host: usize, ) -> Result> { let client = match protocol { Protocol::Grpc => { let decoder: GrpcDecoder = GrpcDecoder::new(send_failed.clone()); - let http_client = Client::build(tls_config, HttpProtocol::Grpc, decoder)?; + let http_client = Client::build( + tls_config, + HttpProtocol::Grpc, + decoder, + pool_idle_timeout, + pool_max_idle_per_host, + )?; UnifiedClientType::Grpc(signing_builder.build(http_client)) } Protocol::Http => { let decoder: HttpDecoder = HttpDecoder::new(send_failed.clone()); - let http_client = Client::build(tls_config, HttpProtocol::Http, decoder)?; + let http_client = Client::build( + tls_config, + HttpProtocol::Http, + decoder, + pool_idle_timeout, + pool_max_idle_per_host, + )?; UnifiedClientType::Http(signing_builder.build(http_client)) } }; diff --git a/src/exporters/otlp/config.rs b/src/exporters/otlp/config.rs index 439e4040..b760e9a4 100644 --- a/src/exporters/otlp/config.rs +++ b/src/exporters/otlp/config.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +use crate::exporters::http::client::{DEFAULT_POOL_IDLE_TIMEOUT, DEFAULT_POOL_MAX_IDLE_PER_HOST}; use crate::exporters::http::retry::RetryConfig; use crate::exporters::http::tls::{Config, ConfigBuilder}; use crate::exporters::otlp::{ @@ -25,6 +26,8 @@ pub struct OTLPExporterConfig { pub(crate) encode_drain_max_time: Duration, pub(crate) export_drain_max_time: Duration, pub(crate) tls_cfg_builder: ConfigBuilder, + pub(crate) pool_idle_timeout: Duration, + pub(crate) pool_max_idle_per_host: usize, } impl OTLPExporterConfig { @@ -46,6 +49,8 @@ impl OTLPExporterConfig { request_timeout: DEFAULT_REQUEST_TIMEOUT, encode_drain_max_time: Duration::from_secs(2), export_drain_max_time: Duration::from_secs(3), + pool_idle_timeout: DEFAULT_POOL_IDLE_TIMEOUT, + pool_max_idle_per_host: DEFAULT_POOL_MAX_IDLE_PER_HOST, } } @@ -122,4 +127,14 @@ impl OTLPExporterConfig { self.export_drain_max_time = max_time; self } + + pub fn with_pool_idle_timeout(mut self, timeout: Duration) -> Self { + self.pool_idle_timeout = timeout; + self + } + + pub fn with_pool_max_idle_per_host(mut self, max_idle: usize) -> Self { + self.pool_max_idle_per_host = max_idle; + self + } } diff --git a/src/exporters/otlp/exporter.rs b/src/exporters/otlp/exporter.rs index ff98456b..031bbf4f 100644 --- a/src/exporters/otlp/exporter.rs +++ b/src/exporters/otlp/exporter.rs @@ -113,6 +113,8 @@ pub fn build_traces_exporter( sent, send_failed.clone(), aws_signing, + traces_config.pool_idle_timeout, + traces_config.pool_max_idle_per_host, )?; let retry_policy = RetryPolicy::new( @@ -223,6 +225,8 @@ pub fn build_logs_exporter( sent, send_failed.clone(), aws_signing, + logs_config.pool_idle_timeout, + logs_config.pool_max_idle_per_host, )?; let retry_policy = RetryPolicy::new( @@ -305,6 +309,8 @@ fn _build_metrics_exporter( sent, send_failed.clone(), aws_signing, + metrics_config.pool_idle_timeout, + metrics_config.pool_max_idle_per_host, )?; let retry_policy = RetryPolicy::new( diff --git a/src/exporters/xray/mod.rs b/src/exporters/xray/mod.rs index a79f6925..7cae9c41 100644 --- a/src/exporters/xray/mod.rs +++ b/src/exporters/xray/mod.rs @@ -103,7 +103,16 @@ impl XRayExporterBuilder { environment: String, creds_provider: AwsCredsProvider, ) -> Result, BoxError> { - let client = Client::build(tls::Config::default(), Protocol::Http, Default::default())?; + use crate::exporters::http::client::{ + DEFAULT_POOL_IDLE_TIMEOUT, DEFAULT_POOL_MAX_IDLE_PER_HOST, + }; + let client = Client::build( + tls::Config::default(), + Protocol::Http, + Default::default(), + DEFAULT_POOL_IDLE_TIMEOUT, + DEFAULT_POOL_MAX_IDLE_PER_HOST, + )?; let transformer = Transformer::new(environment); let req_builder = diff --git a/src/init/otlp_exporter.rs b/src/init/otlp_exporter.rs index ae6147db..68238db7 100644 --- a/src/init/otlp_exporter.rs +++ b/src/init/otlp_exporter.rs @@ -1,3 +1,4 @@ +use crate::exporters::http::client::{DEFAULT_POOL_IDLE_TIMEOUT, DEFAULT_POOL_MAX_IDLE_PER_HOST}; use crate::exporters::otlp; use crate::exporters::otlp::config::OTLPExporterConfig; use crate::exporters::otlp::{CompressionEncoding, Endpoint, Protocol}; @@ -112,6 +113,24 @@ pub struct OTLPExporterBaseArgs { #[serde(with = "humantime_serde")] pub request_timeout: std::time::Duration, + /// Connection pool idle timeout - How long idle connections remain in the pool before being closed. + #[arg( + long("otlp-exporter-pool-idle-timeout"), + env = "ROTEL_OTLP_EXPORTER_POOL_IDLE_TIMEOUT", + default_value = "30s", + value_parser = humantime::parse_duration, + )] + #[serde(with = "humantime_serde")] + pub pool_idle_timeout: Duration, + + /// Maximum idle connections per host - Controls connection reuse for keep-alive. + #[arg( + long("otlp-exporter-pool-max-idle-per-host"), + env = "ROTEL_OTLP_EXPORTER_POOL_MAX_IDLE_PER_HOST", + default_value = "100" + )] + pub pool_max_idle_per_host: usize, + #[command(flatten)] #[serde(flatten)] pub retry: OtlpRetryArgs, @@ -142,6 +161,8 @@ impl Default for OTLPExporterBaseArgs { }, tls_skip_verify: false, request_timeout: Duration::from_secs(5), + pool_idle_timeout: DEFAULT_POOL_IDLE_TIMEOUT, + pool_max_idle_per_host: DEFAULT_POOL_MAX_IDLE_PER_HOST, retry: Default::default(), } } @@ -631,7 +652,9 @@ impl OTLPExporterBaseArgs { .with_tls_skip_verify(self.tls_skip_verify) .with_headers(custom_headers.as_slice()) .with_request_timeout(self.request_timeout.into()) - .with_compression_encoding(self.compression.into()); + .with_compression_encoding(self.compression.into()) + .with_pool_idle_timeout(self.pool_idle_timeout) + .with_pool_max_idle_per_host(self.pool_max_idle_per_host); if let Some(tls_cert_file) = self.cert_group.tls_cert_file { builder = builder.with_cert_file(tls_cert_file.as_str());