From 31483255f6ff7d3e8efc4b8dc0763a66899337d3 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 12 Jun 2026 10:13:49 -0700 Subject: [PATCH 1/4] docs --- README.md | 6 ++++++ src/aws/mod.rs | 3 +++ src/azure/mod.rs | 3 +++ src/gcp/mod.rs | 3 +++ src/http/mod.rs | 3 +++ src/lib.rs | 14 +++++++++++++- 6 files changed, 31 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index da164a7b..02dd8069 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ It's possible to build `object_store` for the `wasm32-unknown-unknown` target, h cargo build -p object_store --target wasm32-unknown-unknown ``` +For `wasm32-wasip1`, where `reqwest` does not compile, use the `*-base` provider features (`aws-base`, `azure-base`, `gcp-base`, `http-base`) and supply your own HTTP client via [`HttpConnector`](https://docs.rs/object_store/latest/object_store/client/trait.HttpConnector.html). + +``` +cargo build -p object_store --no-default-features --features aws-base --target wasm32-wasip1 +``` + ## Related Apache Crates Here are several related crates in different repositories from other Apache projects. diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 42d84a43..c4d8063b 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -17,6 +17,9 @@ //! An object store implementation for S3 //! +//! See the [Feature Flags](crate#feature-flags) section for the difference +//! between the `aws` and `aws-base` features. +//! //! ## Multipart uploads //! //! Multipart uploads can be initiated with the [`ObjectStore::put_multipart_opts`] method. diff --git a/src/azure/mod.rs b/src/azure/mod.rs index 6c4968d6..11567433 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -17,6 +17,9 @@ //! An object store implementation for Azure blob storage //! +//! See the [Feature Flags](crate#feature-flags) section for the difference +//! between the `azure` and `azure-base` features. +//! //! ## Streaming uploads //! //! [`ObjectStore::put_multipart_opts`] will upload data in blocks and write a blob from those blocks. diff --git a/src/gcp/mod.rs b/src/gcp/mod.rs index 3402057c..7c663b86 100644 --- a/src/gcp/mod.rs +++ b/src/gcp/mod.rs @@ -17,6 +17,9 @@ //! An object store implementation for Google Cloud Storage //! +//! See the [Feature Flags](crate#feature-flags) section for the difference +//! between the `gcp` and `gcp-base` features. +//! //! ## Multipart uploads //! //! [Multipart uploads](https://cloud.google.com/storage/docs/multipart-uploads) diff --git a/src/http/mod.rs b/src/http/mod.rs index 3fbf162f..1c83d45b 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -19,6 +19,9 @@ //! //! This follows [rfc2518] commonly known as [WebDAV] //! +//! See the [Feature Flags](crate#feature-flags) section for the difference +//! between the `http` and `http-base` features. +//! //! Basic get support will work out of the box with most HTTP servers, //! even those that don't explicitly support [rfc2518] //! diff --git a/src/lib.rs b/src/lib.rs index 48711bb2..804230c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -96,6 +96,17 @@ doc = "* [`http`]: [HTTP/WebDAV Storage](https://datatracker.ietf.org/doc/html/rfc2518). See [`HttpBuilder`](http::HttpBuilder)" )] //! +//! ## Feature Flags +//! +//! Each cloud provider has two feature flags: a "batteries-included" feature that bundles +//! the default `reqwest`-based HTTP transport, and a `*-base` feature that omits `reqwest` +//! so you can plug in your own HTTP client via [`HttpConnector`](client::HttpConnector). +//! +//! | Feature | Description | +//! | --- | --- | +//! | `aws`, `azure`, `gcp`, `http` | Provider with the built-in `reqwest` HTTP transport. The typical choice. | +//! | `aws-base`, `azure-base`, `gcp-base`, `http-base` | Provider only -- bring your own [`HttpConnector`](client::HttpConnector). Useful when your application already has its own HTTP client, or to keep `reqwest` out of your dependency tree. | +//! //! # Why not a Filesystem Interface? //! //! The [`ObjectStore`] interface is designed to mirror the APIs @@ -533,7 +544,8 @@ //! Many [`ObjectStore`] implementations permit customization of the HTTP client via //! the [`HttpConnector`] trait and utilities in the [`client`] module. //! Examples include injecting custom HTTP headers or using an alternate -//! tokio Runtime I/O requests. +//! tokio Runtime I/O requests. To use a custom connector without bundling +//! `reqwest`, see [Feature Flags](#feature-flags). //! //! [`HttpConnector`]: client::HttpConnector From f51c03bd5e635b9dde7d1dd20a415d522fc21377 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 12 Jun 2026 10:20:16 -0700 Subject: [PATCH 2/4] more docs --- README.md | 15 +++++++++- src/lib.rs | 84 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 02dd8069..77cd61ee 100644 --- a/README.md +++ b/README.md @@ -50,12 +50,25 @@ It's possible to build `object_store` for the `wasm32-unknown-unknown` target, h cargo build -p object_store --target wasm32-unknown-unknown ``` -For `wasm32-wasip1`, where `reqwest` does not compile, use the `*-base` provider features (`aws-base`, `azure-base`, `gcp-base`, `http-base`) and supply your own HTTP client via [`HttpConnector`](https://docs.rs/object_store/latest/object_store/client/trait.HttpConnector.html). +## Disabling `reqwest` + +The `aws`, `azure`, `gcp`, and `http` features each bundle a [`reqwest`]-based HTTP transport. To supply your own HTTP client instead — for example to target [`wasm32-wasip1`] (where `reqwest` does not compile), to share an existing client, or to keep `reqwest` out of your dependency tree — depend on the matching `*-base` feature and provide an [`HttpConnector`] at builder time: + +```toml +[dependencies] +object_store = { version = "0.13", default-features = false, features = ["aws-base"] } +``` ``` cargo build -p object_store --no-default-features --features aws-base --target wasm32-wasip1 ``` +See the [Feature Flags](https://docs.rs/object_store/latest/object_store/#feature-flags) section in the crate docs for the full set of flags. + +[`reqwest`]: https://crates.io/crates/reqwest +[`wasm32-wasip1`]: https://doc.rust-lang.org/rustc/platform-support/wasm32-wasip1.html +[`HttpConnector`]: https://docs.rs/object_store/latest/object_store/client/trait.HttpConnector.html + ## Related Apache Crates Here are several related crates in different repositories from other Apache projects. diff --git a/src/lib.rs b/src/lib.rs index 804230c0..f0e4180c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -96,16 +96,35 @@ doc = "* [`http`]: [HTTP/WebDAV Storage](https://datatracker.ietf.org/doc/html/rfc2518). See [`HttpBuilder`](http::HttpBuilder)" )] //! -//! ## Feature Flags +//! ## Disabling `reqwest` +//! +//! The `aws`, `azure`, `gcp`, and `http` features each bundle a +//! [`reqwest`]-based HTTP transport, which is the right choice for most +//! applications. If you would rather supply your own HTTP client -- for example +//! to share an existing client, to target a platform where `reqwest` does not +//! compile (such as `wasm32-wasip1`), or to keep `reqwest` out of your +//! dependency tree -- use the matching `*-base` feature and provide an +//! [`HttpConnector`](client::HttpConnector) at builder time. +//! +//! ```toml +//! [dependencies] +//! object_store = { version = "0.13", default-features = false, features = ["aws-base"] } +//! ``` //! -//! Each cloud provider has two feature flags: a "batteries-included" feature that bundles -//! the default `reqwest`-based HTTP transport, and a `*-base` feature that omits `reqwest` -//! so you can plug in your own HTTP client via [`HttpConnector`](client::HttpConnector). +//! ```ignore +//! use object_store::aws::AmazonS3Builder; //! -//! | Feature | Description | -//! | --- | --- | -//! | `aws`, `azure`, `gcp`, `http` | Provider with the built-in `reqwest` HTTP transport. The typical choice. | -//! | `aws-base`, `azure-base`, `gcp-base`, `http-base` | Provider only -- bring your own [`HttpConnector`](client::HttpConnector). Useful when your application already has its own HTTP client, or to keep `reqwest` out of your dependency tree. | +//! let store = AmazonS3Builder::from_env() +//! .with_bucket_name("my-bucket") +//! // `my_connector` is your own `impl HttpConnector` +//! .with_http_connector(my_connector) +//! .build()?; +//! ``` +//! +//! See the [Feature Flags](#feature-flags) reference below for the full set +//! of flags. +//! +//! [`reqwest`]: https://crates.io/crates/reqwest //! //! # Why not a Filesystem Interface? //! @@ -545,9 +564,56 @@ //! the [`HttpConnector`] trait and utilities in the [`client`] module. //! Examples include injecting custom HTTP headers or using an alternate //! tokio Runtime I/O requests. To use a custom connector without bundling -//! `reqwest`, see [Feature Flags](#feature-flags). +//! `reqwest`, see [Disabling `reqwest`](#disabling-reqwest). //! //! [`HttpConnector`]: client::HttpConnector +//! +//! # Feature Flags +//! +//! The feature set is layered so that you can pick a provider independently +//! of its HTTP transport: +//! +//! * `cloud-base` holds the shared provider implementation (XML/JSON parsing, +//! credentials, retry, etc.) and intentionally does *not* depend on +//! `reqwest`. +//! * `reqwest` enables the built-in [`reqwest`]-based [`HttpConnector`]. +//! * `-base` (`aws-base`, `azure-base`, `gcp-base`, `http-base`) +//! adds the per-provider logic on top of `cloud-base` without pulling in +//! `reqwest`. +//! * `` (`aws`, `azure`, `gcp`, `http`) is the batteries-included +//! alias for `-base` + `reqwest` and is the typical choice. +//! +//! ## Provider features +//! +//! | Feature | Enables | Notes | +//! | --- | --- | --- | +//! | `aws` | `aws-base` + `reqwest` | Amazon S3 with the built-in HTTP transport. | +//! | `azure` | `azure-base` + `reqwest` | Azure Blob Storage with the built-in HTTP transport. | +//! | `gcp` | `gcp-base` + `reqwest` | Google Cloud Storage with the built-in HTTP transport. | +//! | `http` | `http-base` + `reqwest` | HTTP/WebDAV with the built-in HTTP transport. | +//! | `aws-base` | provider only | S3 provider without `reqwest`; supply your own [`HttpConnector`]. | +//! | `azure-base` | provider only | Azure provider without `reqwest`; supply your own [`HttpConnector`]. | +//! | `gcp-base` | provider only | GCS provider without `reqwest`; supply your own [`HttpConnector`]. | +//! | `http-base` | provider only | HTTP/WebDAV provider without `reqwest`; supply your own [`HttpConnector`]. | +//! +//! ## Transport and shared features +//! +//! | Feature | Description | +//! | --- | --- | +//! | `reqwest` | Enables the default [`reqwest`]-based [`HttpConnector`]. Pulled in automatically by `aws`, `azure`, `gcp`, and `http`. | +//! | `cloud-base` | Shared cloud-provider implementation. Pulled in automatically by every `*-base` feature; usually not enabled directly. | +//! | `cloud` | Back-compat alias for `cloud-base` + `reqwest`. Kept for users that previously depended on the `cloud` umbrella feature. | +//! | `tls-webpki-roots` | When `reqwest` is enabled, also bundle Mozilla's [`webpki-roots`] CA certificates. See [TLS Certificates](#tls-certificates). | +//! +//! ## Other features +//! +//! | Feature | Description | +//! | --- | --- | +//! | `fs` *(default)* | Local filesystem store via [`LocalFileSystem`](local::LocalFileSystem). | +//! | `tokio` | Enables Tokio-based utilities such as [`BufReader`](buffered::BufReader) and [`BufWriter`](buffered::BufWriter). Pulled in automatically by `fs` and the `*-base` features. | +//! | `integration` | Test helpers used by this crate's own integration tests; not intended for downstream use. | +//! +//! [`webpki-roots`]: https://crates.io/crates/webpki-roots #[cfg(feature = "aws-base")] pub mod aws; From 95ac68a99cef12f499be52a66634daa437c0f1a7 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 12 Jun 2026 10:32:15 -0700 Subject: [PATCH 3/4] more docs --- src/lib.rs | 103 +++++++++++++++++++++++++++-------------------------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f0e4180c..79847ceb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -96,35 +96,7 @@ doc = "* [`http`]: [HTTP/WebDAV Storage](https://datatracker.ietf.org/doc/html/rfc2518). See [`HttpBuilder`](http::HttpBuilder)" )] //! -//! ## Disabling `reqwest` -//! -//! The `aws`, `azure`, `gcp`, and `http` features each bundle a -//! [`reqwest`]-based HTTP transport, which is the right choice for most -//! applications. If you would rather supply your own HTTP client -- for example -//! to share an existing client, to target a platform where `reqwest` does not -//! compile (such as `wasm32-wasip1`), or to keep `reqwest` out of your -//! dependency tree -- use the matching `*-base` feature and provide an -//! [`HttpConnector`](client::HttpConnector) at builder time. -//! -//! ```toml -//! [dependencies] -//! object_store = { version = "0.13", default-features = false, features = ["aws-base"] } -//! ``` -//! -//! ```ignore -//! use object_store::aws::AmazonS3Builder; -//! -//! let store = AmazonS3Builder::from_env() -//! .with_bucket_name("my-bucket") -//! // `my_connector` is your own `impl HttpConnector` -//! .with_http_connector(my_connector) -//! .build()?; -//! ``` -//! -//! See the [Feature Flags](#feature-flags) reference below for the full set -//! of flags. -//! -//! [`reqwest`]: https://crates.io/crates/reqwest +//! See [Feature Flags](#feature-flags) for the full set of flags. //! //! # Why not a Filesystem Interface? //! @@ -547,27 +519,6 @@ //! [Apache Iceberg]: https://iceberg.apache.org/ //! [Delta Lake]: https://delta.io/ //! -//! # TLS Certificates -//! -//! Stores that use HTTPS/TLS (this is true for most cloud stores) can choose the source of their [CA] -//! certificates. By default the system-bundled certificates are used (see -//! [`rustls-native-certs`]). The `tls-webpki-roots` feature switch can be used to also bundle Mozilla's -//! root certificates with the library/application (see [`webpki-roots`]). -//! -//! [CA]: https://en.wikipedia.org/wiki/Certificate_authority -//! [`rustls-native-certs`]: https://crates.io/crates/rustls-native-certs/ -//! [`webpki-roots`]: https://crates.io/crates/webpki-roots -//! -//! # Customizing HTTP Clients -//! -//! Many [`ObjectStore`] implementations permit customization of the HTTP client via -//! the [`HttpConnector`] trait and utilities in the [`client`] module. -//! Examples include injecting custom HTTP headers or using an alternate -//! tokio Runtime I/O requests. To use a custom connector without bundling -//! `reqwest`, see [Disabling `reqwest`](#disabling-reqwest). -//! -//! [`HttpConnector`]: client::HttpConnector -//! //! # Feature Flags //! //! The feature set is layered so that you can pick a provider independently @@ -611,9 +562,59 @@ //! | --- | --- | //! | `fs` *(default)* | Local filesystem store via [`LocalFileSystem`](local::LocalFileSystem). | //! | `tokio` | Enables Tokio-based utilities such as [`BufReader`](buffered::BufReader) and [`BufWriter`](buffered::BufWriter). Pulled in automatically by `fs` and the `*-base` features. | -//! | `integration` | Test helpers used by this crate's own integration tests; not intended for downstream use. | +//! | `integration` | Exposes the [`integration`] module, a reusable test suite for verifying custom [`ObjectStore`] implementations. Not API-stable. | //! +//! # TLS Certificates +//! +//! Stores that use HTTPS/TLS (this is true for most cloud stores) can choose the source of their [CA] +//! certificates. By default the system-bundled certificates are used (see +//! [`rustls-native-certs`]). The `tls-webpki-roots` feature switch can be used to also bundle Mozilla's +//! root certificates with the library/application (see [`webpki-roots`]). +//! +//! [CA]: https://en.wikipedia.org/wiki/Certificate_authority +//! [`rustls-native-certs`]: https://crates.io/crates/rustls-native-certs/ //! [`webpki-roots`]: https://crates.io/crates/webpki-roots +//! +//! # Customizing HTTP Clients +//! +//! Many [`ObjectStore`] implementations permit customization of the HTTP client via +//! the [`HttpConnector`] trait and utilities in the [`client`] module. +//! Examples include injecting custom HTTP headers or using an alternate +//! tokio Runtime for I/O requests. To replace `reqwest` entirely (rather than +//! tweak the bundled transport) see [Disabling `reqwest`](#disabling-reqwest). +//! +//! [`HttpConnector`]: client::HttpConnector +//! +//! # Disabling `reqwest` +//! +//! The `aws`, `azure`, `gcp`, and `http` features each bundle a +//! [`reqwest`]-based HTTP transport, which is the right choice for most +//! applications. If you would rather supply your own HTTP client — for example +//! to share an existing client, to target a platform where `reqwest` does not +//! compile (such as `wasm32-wasip1`), or to keep `reqwest` out of your +//! dependency tree — use the matching `*-base` feature and provide an +//! [`HttpConnector`](client::HttpConnector) at builder time. +//! +//! Remember to disable the default features so that `fs` (and its transitive +//! dependencies) is not pulled in: +//! +//! ```toml +//! [dependencies] +//! object_store = { version = "0.13", default-features = false, features = ["aws-base"] } +//! ``` +//! +//! ```ignore +//! use object_store::aws::AmazonS3Builder; +//! +//! let store = AmazonS3Builder::from_env() +//! // `my_connector` is your own `impl HttpConnector` +//! .with_http_connector(my_connector) +//! .build()?; +//! ``` +//! +//! See [Feature Flags](#feature-flags) above for the full set of flags. +//! +//! [`reqwest`]: https://crates.io/crates/reqwest #[cfg(feature = "aws-base")] pub mod aws; From 4bb464405d76d597244f0bb2aefb27499790fe83 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 12 Jun 2026 11:29:41 -0700 Subject: [PATCH 4/4] minor changes --- README.md | 11 +++-------- src/lib.rs | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 77cd61ee..80c49474 100644 --- a/README.md +++ b/README.md @@ -52,22 +52,17 @@ cargo build -p object_store --target wasm32-unknown-unknown ## Disabling `reqwest` -The `aws`, `azure`, `gcp`, and `http` features each bundle a [`reqwest`]-based HTTP transport. To supply your own HTTP client instead — for example to target [`wasm32-wasip1`] (where `reqwest` does not compile), to share an existing client, or to keep `reqwest` out of your dependency tree — depend on the matching `*-base` feature and provide an [`HttpConnector`] at builder time: - -```toml -[dependencies] -object_store = { version = "0.13", default-features = false, features = ["aws-base"] } -``` +The `aws`, `azure`, `gcp`, and `http` features each bundle a [`reqwest`]-based HTTP transport. To target [`wasm32-wasip1`] (where `reqwest` does not compile) or otherwise supply your own HTTP client, build against the matching `*-base` feature instead: ``` cargo build -p object_store --no-default-features --features aws-base --target wasm32-wasip1 ``` -See the [Feature Flags](https://docs.rs/object_store/latest/object_store/#feature-flags) section in the crate docs for the full set of flags. +See [Disabling `reqwest`] in the crate docs for full details. [`reqwest`]: https://crates.io/crates/reqwest [`wasm32-wasip1`]: https://doc.rust-lang.org/rustc/platform-support/wasm32-wasip1.html -[`HttpConnector`]: https://docs.rs/object_store/latest/object_store/client/trait.HttpConnector.html +[Disabling `reqwest`]: https://docs.rs/object_store/latest/object_store/#disabling-reqwest ## Related Apache Crates diff --git a/src/lib.rs b/src/lib.rs index 79847ceb..b2b199ee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -552,9 +552,9 @@ //! | Feature | Description | //! | --- | --- | //! | `reqwest` | Enables the default [`reqwest`]-based [`HttpConnector`]. Pulled in automatically by `aws`, `azure`, `gcp`, and `http`. | +//! | `tls-webpki-roots` | When `reqwest` is enabled, also bundle Mozilla's [`webpki-roots`] CA certificates. See [TLS Certificates](#tls-certificates). | //! | `cloud-base` | Shared cloud-provider implementation. Pulled in automatically by every `*-base` feature; usually not enabled directly. | //! | `cloud` | Back-compat alias for `cloud-base` + `reqwest`. Kept for users that previously depended on the `cloud` umbrella feature. | -//! | `tls-webpki-roots` | When `reqwest` is enabled, also bundle Mozilla's [`webpki-roots`] CA certificates. See [TLS Certificates](#tls-certificates). | //! //! ## Other features //!