Make reqwest optional (#724)

* makes reqwest optional

* refactor!: make aws/azure/gcp/http depend on reqwest, not cloud

The role of cloud has been taken over by reqwest + cloud-base.

Implements #724 (comment)

BREAKING CHANGE: according to cargo semver-checks: aws/azure/gcp/http no longer enable the cloud feature.

* document feature flags exactly as kevinjqliu proposed

* switch from `feature="cloud"` to `feature="reqwest"`

* Apply suggestion from @kevinjqliu

Co-authored-by: Kevin Liu <kevinjqliu@users.noreply.github.com>

---------

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
Co-authored-by: Kyle Barron <kyle@developmentseed.org>
Co-authored-by: Kyle Barron <kylebarron2@gmail.com>
Co-authored-by: Kevin Liu <kevinjqliu@users.noreply.github.com>

Author: 4 people

.github/workflows/ci.yml

@@ -57,6 +57,26 @@ jobs:
         run: cargo clippy --features azure -- -D warnings
       - name: Run clippy with http feature
         run: cargo clippy --features http -- -D warnings
+      - name: Run clippy with aws-base feature
+        run: cargo clippy --no-default-features --features aws-base -- -D warnings
+      - name: Run clippy with azure-base feature
+        run: cargo clippy --no-default-features --features azure-base -- -D warnings
+      - name: Run clippy with gcp-base feature
+        run: cargo clippy --no-default-features --features gcp-base -- -D warnings
+      - name: Run clippy with http-base feature
+        run: cargo clippy --no-default-features --features http-base -- -D warnings
+      - name: Verify base features do not enable reqwest
+        run: |
+          if cargo tree \
+            --no-default-features \
+            --features aws-base,azure-base,gcp-base,http-base \
+            --edges normal \
+            --prefix none \
+            -f '{p}' \
+            | grep -E '^reqwest v'; then
+            echo "reqwest must not be enabled by *-base features"
+            exit 1
+          fi
       - name: Run clippy with integration feature
         run: cargo clippy --no-default-features --features integration -- -D warnings
       - name: Run clippy with all features
@@ -196,6 +216,8 @@ jobs:
         run: rustup target add wasm32-wasip1
       - name: Build wasm32-wasip1
         run: cargo build --all-features --target wasm32-wasip1
+      - name: Build wasm32-wasip1 without reqwest
+        run: cargo build --no-default-features --features aws-base --target wasm32-wasip1
       - name: Install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - uses: actions/setup-node@v6

Cargo.toml

@@ -82,12 +82,35 @@ futures-channel = {version = "0.3", features = ["sink"]}
 
 [features]
 default = ["fs"]
-cloud = ["serde", "serde_json", "quick-xml", "hyper", "reqwest", "reqwest/stream", "chrono/serde", "base64", "rand", "ring", "http-body-util", "form_urlencoded", "serde_urlencoded", "tokio"]
-azure = ["cloud", "httparse"]
+
+# Shared cloud/provider implementation dependencies.
+# This intentionally does NOT include reqwest.
+cloud-base = ["serde", "serde_json", "quick-xml", "hyper", "chrono/serde", "base64", "rand", "ring", "http-body-util", "form_urlencoded", "serde_urlencoded", "tokio"]
+
+# Built-in reqwest-based HTTP transport.
+reqwest = ["dep:reqwest", "reqwest/stream"]
+
+# Compatibility/convenience alias.
+# Historically, cloud meant "common cloud support", including reqwest.
+# Keep that behavior for existing users.
+cloud = ["cloud-base", "reqwest"]
+
+# Provider base features.
+# These compile provider logic without forcing reqwest.
+azure-base = ["cloud-base", "httparse"]
+gcp-base = ["cloud-base", "rustls-pki-types"]
+aws-base = ["cloud-base", "crc-fast", "md-5"]
+http-base = ["cloud-base"]
+
+# Batteries-included provider features.
+# These preserve existing behavior: enabling aws/azure/gcp/http gives
+# the default reqwest transport.
+azure = ["azure-base", "reqwest"]
+gcp = ["gcp-base", "reqwest"]
+aws = ["aws-base", "reqwest"]
+http = ["http-base", "reqwest"]
+
 fs = ["walkdir", "tokio", "nix", "windows-sys"]
-gcp = ["cloud", "rustls-pki-types"]
-aws = ["cloud", "crc-fast", "md-5"]
-http = ["cloud"]
 tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"]
 integration = ["rand", "tokio"]
 tokio = ["dep:tokio", "dep:tracing"]

src/aws/builder.rs

@@ -29,9 +29,9 @@ use crate::config::ConfigValue;
 use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
 use base64::Engine;
 use base64::prelude::BASE64_STANDARD;
+use http::header::{HeaderMap, HeaderValue};
 use itertools::Itertools;
 use md5::{Digest, Md5};
-use reqwest::header::{HeaderMap, HeaderValue};
 use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 use std::sync::Arc;
@@ -1573,6 +1573,7 @@ mod tests {
         assert!(builder.unsigned_payload.get().unwrap());
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn s3_test_endpoint_url_s3_config() {
         // Verify aws_endpoint_url_s3 parses to S3Endpoint config key
@@ -1687,6 +1688,7 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn s3_default_region() {
         let builder = AmazonS3Builder::new()
@@ -1696,6 +1698,7 @@ mod tests {
         assert_eq!(builder.client.config.region, "us-east-1");
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn s3_test_bucket_endpoint() {
         let builder = AmazonS3Builder::new()
@@ -1795,6 +1798,7 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn s3_test_proxy_url() {
         let s3 = AmazonS3Builder::new()
@@ -1823,6 +1827,7 @@ mod tests {
         assert_eq!("Generic HTTP client error: builder error", err);
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn test_invalid_config() {
         let err = AmazonS3Builder::new()
@@ -1865,6 +1870,7 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn test_request_payer_config() {
         let s3 = AmazonS3Builder::new()
@@ -1938,6 +1944,7 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn test_builder_eks_with_config() {
         let builder = AmazonS3Builder::new()
@@ -1960,6 +1967,7 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn test_builder_web_identity_with_config() {
         let builder = AmazonS3Builder::new()

src/aws/client.rs

@@ -1041,6 +1041,7 @@ mod tests {
     use hyper::Request;
     use hyper::body::Incoming;
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_create_multipart_has_content_length() {
         let mock = MockServer::new().await;
@@ -1138,6 +1139,7 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_default_headers_signed_request() {
         let mock = MockServer::new().await;
@@ -1162,6 +1164,7 @@ mod tests {
         mock.shutdown().await;
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_default_headers_signed_bulk_delete() {
         let mock = MockServer::new().await;
@@ -1338,6 +1341,7 @@ mod tests {
         mock.shutdown().await;
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_default_headers_signed_get_request() {
         let mock = MockServer::new().await;
@@ -1360,6 +1364,7 @@ mod tests {
         mock.shutdown().await;
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_default_headers_signed_complete_multipart() {
         let mock = MockServer::new().await;

src/aws/credential.rs

@@ -865,11 +865,13 @@ mod tests {
     use crate::aws::{AmazonS3Builder, AmazonS3ConfigKey};
     use crate::client::HttpClient;
     use crate::client::mock_server::MockServer;
-    use http::Response;
-    use reqwest::{Client, Method};
+    use http::{Method, Response};
+    #[cfg(feature = "reqwest")]
+    use reqwest::Client;
     use std::env;
 
     // Test generated using https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+    #[cfg(feature = "reqwest")]
     #[test]
     fn test_sign_with_signed_payload() {
         let client = HttpClient::new(Client::new());
@@ -914,6 +916,7 @@ mod tests {
         )
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn test_sign_with_signed_payload_request_payer() {
         let client = HttpClient::new(Client::new());
@@ -958,6 +961,7 @@ mod tests {
         )
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn test_sign_with_unsigned_payload() {
         let client = HttpClient::new(Client::new());
@@ -1085,6 +1089,7 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "reqwest")]
     #[test]
     fn test_sign_port() {
         let client = HttpClient::new(Client::new());
@@ -1128,6 +1133,7 @@ mod tests {
         )
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_instance_metadata() {
         if env::var("TEST_INTEGRATION").is_err() {
@@ -1166,6 +1172,7 @@ mod tests {
         assert!(!token.is_empty())
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_mock() {
         let server = MockServer::new().await;
@@ -1259,6 +1266,7 @@ mod tests {
             .unwrap_err();
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_eks_pod_credential_provider() {
         use crate::client::mock_server::MockServer;

src/aws/mod.rs

@@ -31,8 +31,8 @@
 use async_trait::async_trait;
 use futures_util::stream::BoxStream;
 use futures_util::{StreamExt, TryStreamExt};
-use reqwest::header::{HeaderName, IF_MATCH, IF_NONE_MATCH};
-use reqwest::{Method, StatusCode};
+use http::header::{HeaderName, IF_MATCH, IF_NONE_MATCH};
+use http::{Method, StatusCode};
 use std::{sync::Arc, time::Duration};
 use url::Url;
 
@@ -58,14 +58,14 @@ mod client;
 mod credential;
 mod precondition;
 
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(all(feature = "reqwest", not(target_arch = "wasm32")))]
 mod resolve;
 
 pub use builder::{AmazonS3Builder, AmazonS3ConfigKey};
 pub use checksum::Checksum;
 pub use precondition::{S3ConditionalPut, S3CopyIfNotExists};
 
-#[cfg(not(target_arch = "wasm32"))]
+#[cfg(all(feature = "reqwest", not(target_arch = "wasm32")))]
 pub use resolve::resolve_bucket_region;
 
 /// This struct is used to maintain the URI path encoding
@@ -118,7 +118,7 @@ impl Signer for AmazonS3 {
     /// ```
     /// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
     /// # use object_store::{aws::AmazonS3Builder, path::Path, signer::Signer};
-    /// # use reqwest::Method;
+    /// # use http::Method;
     /// # use std::time::Duration;
     /// #
     /// let region = "us-east-1";
@@ -534,6 +534,7 @@ mod tests {
     use super::*;
     use crate::ClientOptions;
     use crate::ObjectStoreExt;
+    #[cfg(feature = "reqwest")]
     use crate::client::SpawnedReqwestConnector;
     use crate::client::get::GetClient;
     use crate::client::retry::RetryContext;
@@ -946,6 +947,7 @@ mod tests {
 
     /// Integration test that ensures I/O is done on an alternate threadpool
     /// when using the `SpawnedReqwestConnector`.
+    #[cfg(feature = "reqwest")]
     #[test]
     fn s3_alternate_threadpool_spawned_request_connector() {
         maybe_skip_integration!();

src/aws/precondition.rs

@@ -44,7 +44,7 @@ pub enum S3CopyIfNotExists {
     /// other than 412.
     ///
     /// Encoded as `header-with-status:<HEADER_NAME>:<HEADER_VALUE>:<STATUS>` ignoring whitespace
-    HeaderWithStatus(String, String, reqwest::StatusCode),
+    HeaderWithStatus(String, String, http::StatusCode),
     /// Native Amazon S3 supports copy if not exists through a multipart upload
     /// where the upload copies an existing object and is completed only if the
     /// new object does not already exist.
@@ -180,7 +180,7 @@ mod tests {
         let expected = Some(S3CopyIfNotExists::HeaderWithStatus(
             "key".to_owned(),
             "value".to_owned(),
-            reqwest::StatusCode::FORBIDDEN,
+            http::StatusCode::FORBIDDEN,
         ));
 
         assert_eq!(expected, S3CopyIfNotExists::from_str(input));
@@ -212,7 +212,7 @@ mod tests {
         let expected = Some(S3CopyIfNotExists::HeaderWithStatus(
             "key".to_owned(),
             "value".to_owned(),
-            reqwest::StatusCode::FORBIDDEN,
+            http::StatusCode::FORBIDDEN,
         ));
 
         const INPUTS: &[&str] = &[

src/aws/resolve.rs

@@ -47,7 +47,7 @@ impl From<Error> for crate::Error {
 ///
 /// [HeadBucket API]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html
 pub async fn resolve_bucket_region(bucket: &str, client_options: &ClientOptions) -> Result<String> {
-    use reqwest::StatusCode;
+    use http::StatusCode;
 
     let endpoint = format!("https://{bucket}.s3.amazonaws.com");
 

src/azure/client.rs

@@ -1385,6 +1385,7 @@ mod tests {
             quick_xml::de::from_str(S).unwrap();
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_build_bulk_delete_body() {
         let credential_provider = Arc::new(StaticCredentialProvider::new(

src/azure/credential.rs

@@ -1075,6 +1075,7 @@ mod tests {
     use crate::client::mock_server::MockServer;
     use crate::{ObjectStoreExt, Path};
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_managed_identity() {
         let server = MockServer::new().await;
@@ -1133,6 +1134,7 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_workload_identity() {
         let server = MockServer::new().await;
@@ -1185,6 +1187,7 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_no_credentials() {
         let server = MockServer::new().await;
@@ -1218,6 +1221,7 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "reqwest")]
     #[tokio::test]
     async fn test_fabric_refresh_expired_token() {
         let server = MockServer::new().await;