feat(http-client): add knob to disable connection pooling (#1933)

# What does this PR do?

This PR adds a configuration knob to disable connection pooling for the HTTP client common component. This doesn't assume that the backend has specific support for connection pooling, but allows to say "if you have connection pooling, do not use it".

# Motivation

This PR has been split off #1806, which builds an agent-level HTTP client on top of the basic HTTP client. To communicate with the agent, the current behavior of existing helpers in e.g. libdd-common is to disable connection pooling by default, because the agent has a low keep-alive timeout and the most common case is to flush data on a periodic schedule. There, connection pooling is detrimental as the connection will be invalidated as soon as it is re-used, see https://github.com/DataDog/libdatadog/blob/cff72917bfe6a4f916db03294ae1ae895f58b882/libdd-common/src/http_common.rs#L118-L128.

# Additional Notes

N/A

# How to test the change?

N/A

Co-authored-by: yann.hamdaoui <yann.hamdaoui@datadoghq.com>

Author: yannham

libdd-http-client/src/backend/hyper_backend.rs

@@ -145,11 +145,19 @@ fn map_hyper_error(e: hyper_util::client::legacy::Error) -> HttpClientError {
 
 impl super::Backend for HyperBackend {
     fn new(
-        _timeout: std::time::Duration,
+        client_config: &HttpClientConfig,
         transport: TransportConfig,
     ) -> Result<Self, HttpClientError> {
-        let client = http_common::client_builder().build(Connector::default());
-        Ok(Self { client, transport })
+        let mut builder = http_common::client_builder();
+
+        if !client_config.allow_connection_pooling() {
+            builder.pool_max_idle_per_host(0);
+        }
+
+        Ok(Self {
+            client: builder.build(Connector::default()),
+            transport,
+        })
     }
 
     async fn send(

libdd-http-client/src/backend/mod.rs

@@ -1,8 +1,7 @@
 // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::{config, HttpClientError, HttpRequest, HttpResponse};
-use std::time::Duration;
+use crate::{config, HttpClientConfig, HttpClientError, HttpRequest, HttpResponse};
 
 #[cfg(all(feature = "hyper-backend", not(feature = "reqwest-backend")))]
 pub(crate) mod hyper_backend;
@@ -16,7 +15,10 @@ pub(crate) mod reqwest_backend;
 /// Backend`.
 pub(crate) trait Backend: Sized {
     /// Construct a new backend with the given timeout and transport.
-    fn new(timeout: Duration, transport: config::TransportConfig) -> Result<Self, HttpClientError>;
+    fn new(
+        client_config: &HttpClientConfig,
+        transport: config::TransportConfig,
+    ) -> Result<Self, HttpClientError>;
 
     /// Send an HTTP request and return the response.
     async fn send(

libdd-http-client/src/backend/reqwest_backend.rs

@@ -18,10 +18,10 @@ pub(crate) struct ReqwestBackend {
 impl super::Backend for ReqwestBackend {
     // Creates a `reqwest::Client` with connection pooling enabled.
     fn new(
-        timeout: std::time::Duration,
+        client_config: &HttpClientConfig,
         transport: TransportConfig,
     ) -> Result<Self, HttpClientError> {
-        let mut builder = reqwest::Client::builder().timeout(timeout);
+        let mut builder = reqwest::Client::builder().timeout(client_config.timeout());
 
         match transport {
             TransportConfig::Tcp => {}
@@ -35,6 +35,10 @@ impl super::Backend for ReqwestBackend {
             }
         }
 
+        if !client_config.allow_connection_pooling() {
+            builder = builder.pool_max_idle_per_host(0);
+        }
+
         let client = builder
             .build()
             .map_err(|e| HttpClientError::InvalidConfig(e.to_string()))?;

libdd-http-client/src/client.rs

@@ -46,8 +46,10 @@ impl HttpClient {
         config: HttpClientConfig,
         transport: TransportConfig,
     ) -> Result<Self, HttpClientError> {
-        let backend = BackendImpl::new(config.timeout(), transport)?;
-        Ok(Self { backend, config })
+        Ok(Self {
+            backend: BackendImpl::new(&config, transport)?,
+            config,
+        })
     }
 
     /// The client's configuration.

libdd-http-client/src/config.rs

@@ -33,6 +33,7 @@ pub struct HttpClientConfig {
     timeout: Duration,
     treat_http_errors_as_errors: bool,
     retry: Option<RetryConfig>,
+    allow_connection_pooling: bool,
 }
 
 impl HttpClientConfig {
@@ -44,6 +45,7 @@ impl HttpClientConfig {
             timeout,
             treat_http_errors_as_errors: true,
             retry: None,
+            allow_connection_pooling: true,
         }
     }
 
@@ -66,28 +68,45 @@ impl HttpClientConfig {
     pub fn retry(&self) -> Option<&RetryConfig> {
         self.retry.as_ref()
     }
+
+    /// Whether connection pooling can be used, when available. See
+    /// [HttpClientBuilder::allow_connection_pooling].
+    pub fn allow_connection_pooling(&self) -> bool {
+        self.allow_connection_pooling
+    }
 }
 
 /// Builder for [`crate::HttpClient`].
 ///
 /// Obtain via [`crate::HttpClient::builder`].
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct HttpClientBuilder {
     base_url: Option<String>,
     timeout: Option<Duration>,
     treat_http_errors_as_errors: bool,
     retry: Option<RetryConfig>,
     transport: TransportConfig,
+    allow_connection_pooling: bool,
 }
 
-impl HttpClientBuilder {
-    /// Create a new builder with default settings.
-    pub fn new() -> Self {
+impl Default for HttpClientBuilder {
+    fn default() -> Self {
         Self {
+            base_url: Default::default(),
+            timeout: Default::default(),
             treat_http_errors_as_errors: true,
-            ..Default::default()
+            retry: Default::default(),
+            transport: Default::default(),
+            allow_connection_pooling: true,
         }
     }
+}
+
+impl HttpClientBuilder {
+    /// Create a new builder with default settings.
+    pub fn new() -> Self {
+        Self::default()
+    }
 
     /// Set the base URL.
     pub fn base_url(mut self, url: String) -> Self {
@@ -136,6 +155,18 @@ impl HttpClientBuilder {
         self
     }
 
+    /// Allow connection pooling. Defaults to `true`.
+    ///
+    /// Note that whether pooling is actually used depends on the HTTP backend of
+    /// [crate], though both currently available backends (reqwest and hyper) support
+    /// pooling. This setting should be understood as: if set to `true`, the default behavior of the
+    /// underlying backend will be selected, which might or might not do connection pooling by
+    /// default. If set to `false`, we guarantee no connection pooling will happen.
+    pub fn allow_connection_pooling(mut self, allow: bool) -> Self {
+        self.allow_connection_pooling = allow;
+        self
+    }
+
     /// Build the [`crate::HttpClient`].
     ///
     /// Returns [`crate::HttpClientError::InvalidConfig`] if required fields
@@ -152,6 +183,7 @@ impl HttpClientBuilder {
             timeout,
             treat_http_errors_as_errors: self.treat_http_errors_as_errors,
             retry: self.retry,
+            allow_connection_pooling: self.allow_connection_pooling,
         };
         crate::HttpClient::from_config_and_transport(config, self.transport)
     }
@@ -230,4 +262,27 @@ mod tests {
             .unwrap();
         assert!(!client.config().treat_http_errors_as_errors());
     }
+
+    #[test]
+    fn builder_allow_connection_pooling_defaults_true() {
+        ensure_crypto_provider();
+        let client = HttpClientBuilder::new()
+            .base_url("http://localhost".to_owned())
+            .timeout(Duration::from_secs(1))
+            .build()
+            .unwrap();
+        assert!(client.config().allow_connection_pooling());
+    }
+
+    #[test]
+    fn builder_allow_connection_pooling_set_false() {
+        ensure_crypto_provider();
+        let client = HttpClientBuilder::new()
+            .base_url("http://localhost".to_owned())
+            .timeout(Duration::from_secs(1))
+            .allow_connection_pooling(false)
+            .build()
+            .unwrap();
+        assert!(!client.config().allow_connection_pooling());
+    }
 }