From 5e3a4cf56bafeb8013e142ef61edfa274173f55c Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Fri, 22 May 2026 20:21:06 -0700 Subject: [PATCH 01/14] [capabilities] add capabilities method to ObjectStore trait I'm starting with a single capability for ordered list results. GCP and Azure list objects always return ordered results. For AWS it depends on a bucket type, for directory buckets results are not ordered. I'm thinking about adding new config option for indicating whether S3 bucket is a directory bucket or not. But for now we can say that AWS results are not ordered. --- src/aws/builder.rs | 22 ++++++++++++++++++++-- src/aws/mod.rs | 17 ++++++++++++----- src/azure/builder.rs | 22 ++++++++++++++++++++-- src/azure/mod.rs | 17 ++++++++++++++--- src/gcp/builder.rs | 16 +++++++++++++++- src/gcp/mod.rs | 18 +++++++++++++++--- src/integration.rs | 12 ++++++++++++ src/lib.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/memory.rs | 12 +++++++++--- 9 files changed, 161 insertions(+), 19 deletions(-) diff --git a/src/aws/builder.rs b/src/aws/builder.rs index 5a6d2b49..7aa50df0 100644 --- a/src/aws/builder.rs +++ b/src/aws/builder.rs @@ -26,7 +26,9 @@ use crate::aws::{ }; use crate::client::{HttpConnector, TokenCredentialProvider, http_connector}; use crate::config::ConfigValue; -use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider}; +use crate::{ + Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider, +}; use base64::Engine; use base64::prelude::BASE64_STANDARD; use itertools::Itertools; @@ -193,6 +195,8 @@ pub struct AmazonS3Builder { request_payer: ConfigValue, /// The [`HttpConnector`] to use http_connector: Option>, + /// Capabilities to advertise for this store instance + capabilities: Option, } /// Configuration keys for [`AmazonS3Builder`] @@ -1105,6 +1109,17 @@ impl AmazonS3Builder { self } + /// Override the [`Capabilities`] advertised by this store. + /// + /// By default the store reports `ordered_listing: true` because S3 + /// `ListObjectsV2` returns results in lexicographic order. Use this + /// method if you are connecting to an S3-compatible endpoint whose + /// behaviour differs from the standard S3 API. + pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { + self.capabilities = Some(capabilities); + self + } + /// Create a [`AmazonS3`] instance from the provided values, /// consuming `self`. pub fn build(mut self) -> Result { @@ -1286,7 +1301,10 @@ impl AmazonS3Builder { let http_client = http.connect(&config.client_options)?; let client = Arc::new(S3Client::new(config, http_client)); - Ok(AmazonS3 { client }) + Ok(AmazonS3 { + client, + capabilities: self.capabilities, + }) } } diff --git a/src/aws/mod.rs b/src/aws/mod.rs index e1cdb065..730415be 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -43,11 +43,7 @@ use crate::client::list::{ListClient, ListClientExt}; use crate::multipart::{MultipartStore, PartId}; use crate::signer::Signer; use crate::util::STRICT_ENCODE_SET; -use crate::{ - CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, - ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult, - Result, UploadPart, -}; +use crate::{Capabilities, CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart}; static TAGS_HEADER: HeaderName = HeaderName::from_static("x-amz-tagging"); static COPY_SOURCE_HEADER: HeaderName = HeaderName::from_static("x-amz-copy-source"); @@ -79,10 +75,17 @@ use crate::client::parts::Parts; use crate::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; pub use credential::{AwsAuthorizer, AwsCredential}; +pub fn get_default_capabilities() -> Capabilities { + Capabilities { + ordered_listing: false, + } +} + /// Interface for [Amazon S3](https://aws.amazon.com/s3/). #[derive(Debug, Clone)] pub struct AmazonS3 { client: Arc, + capabilities: Option, } impl std::fmt::Display for AmazonS3 { @@ -413,6 +416,10 @@ impl ObjectStore for AmazonS3 { } } } + + fn capabilities(&self) -> Capabilities { + self.capabilities.or_else(get_default_capabilities) + } } #[derive(Debug)] diff --git a/src/azure/builder.rs b/src/azure/builder.rs index 1f57fac5..a5262807 100644 --- a/src/azure/builder.rs +++ b/src/azure/builder.rs @@ -23,7 +23,9 @@ use crate::azure::credential::{ use crate::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzure, STORE}; use crate::client::{HttpConnector, TokenCredentialProvider, http_connector}; use crate::config::ConfigValue; -use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider}; +use crate::{ + Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider, +}; use percent_encoding::percent_decode_str; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -180,6 +182,8 @@ pub struct MicrosoftAzureBuilder { fabric_cluster_identifier: Option, /// The [`HttpConnector`] to use http_connector: Option>, + /// Capabilities to advertise for this store instance + capabilities: Option, } /// Configuration keys for [`MicrosoftAzureBuilder`] @@ -906,6 +910,17 @@ impl MicrosoftAzureBuilder { self } + /// Override the [`Capabilities`] advertised by this store. + /// + /// By default the store reports `ordered_listing: true` because Azure Blob + /// Storage returns list results in lexicographic order. Use this method if + /// you are connecting to an endpoint whose behaviour differs from the + /// standard Azure Blob Storage API. + pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { + self.capabilities = capabilities; + self + } + /// Configure a connection to container with given name on Microsoft Azure Blob store. pub fn build(mut self) -> Result { if let Some(url) = self.url.take() { @@ -1054,7 +1069,10 @@ impl MicrosoftAzureBuilder { let http_client = http.connect(&config.client_options)?; let client = Arc::new(AzureClient::new(config, http_client)); - Ok(MicrosoftAzure { client }) + Ok(MicrosoftAzure { + client, + capabilities: self.capabilities, + }) } } diff --git a/src/azure/mod.rs b/src/azure/mod.rs index 1429bec9..205e5d48 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -24,9 +24,9 @@ //! Unused blocks will automatically be dropped after 7 days. //! use crate::{ - CopyMode, CopyOptions, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, - ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, - UploadPart, + Capabilities, CopyMode, CopyOptions, GetOptions, GetResult, ListResult, MultipartId, + MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, + PutResult, Result, UploadPart, multipart::{MultipartStore, PartId}, path::Path, signer::Signer, @@ -58,10 +58,17 @@ pub use credential::AzureCredential; const STORE: &str = "MicrosoftAzure"; +pub fn get_default_capabilities() -> Capabilities { + Capabilities { + ordered_listing: true, + } +} + /// Interface for [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/). #[derive(Debug)] pub struct MicrosoftAzure { client: Arc, + capabilities: Option, } impl MicrosoftAzure { @@ -180,6 +187,10 @@ impl ObjectStore for MicrosoftAzure { CopyMode::Create => self.client.copy_request(from, to, false).await, } } + + fn capabilities(&self) -> Capabilities { + self.capabilities.or_else(get_default_capabilities) + } } #[async_trait] diff --git a/src/gcp/builder.rs b/src/gcp/builder.rs index 82752b05..f24100cb 100644 --- a/src/gcp/builder.rs +++ b/src/gcp/builder.rs @@ -26,7 +26,7 @@ use crate::gcp::{ GcpCredential, GcpCredentialProvider, GcpSigningCredential, GcpSigningCredentialProvider, GoogleCloudStorage, STORE, credential, }; -use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider}; +use crate::{Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider}; use serde::{Deserialize, Serialize}; use std::str::FromStr; use std::sync::Arc; @@ -120,6 +120,8 @@ pub struct GoogleCloudStorageBuilder { signing_credentials: Option, /// The [`HttpConnector`] to use http_connector: Option>, + /// Capabilities to advertise for this store instance + capabilities: Option, } /// Configuration keys for [`GoogleCloudStorageBuilder`] @@ -534,6 +536,17 @@ impl GoogleCloudStorageBuilder { self } + /// Override the [`Capabilities`] advertised by this store. + /// + /// By default the store reports `ordered_listing: true` because GCS + /// returns list results in lexicographic order. Use this method if you + /// are connecting to an endpoint whose behaviour differs from the + /// standard GCS API. + pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { + self.capabilities = Some(capabilities); + self + } + /// Configure a connection to Google Cloud Storage, returning a /// new [`GoogleCloudStorage`] and consuming `self` pub fn build(mut self) -> Result { @@ -669,6 +682,7 @@ impl GoogleCloudStorageBuilder { let http_client = http.connect(&config.client_options)?; Ok(GoogleCloudStorage { client: Arc::new(GoogleCloudStorageClient::new(config, http_client)?), + capabilities: self.capabilities, }) } } diff --git a/src/gcp/mod.rs b/src/gcp/mod.rs index 51e85ae6..21d5e27e 100644 --- a/src/gcp/mod.rs +++ b/src/gcp/mod.rs @@ -42,9 +42,9 @@ use crate::client::CredentialProvider; use crate::gcp::credential::GCSAuthorizer; use crate::signer::Signer; use crate::{ - GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore, - PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart, multipart::PartId, - path::Path, + Capabilities, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, + ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart, + multipart::PartId, path::Path, }; use async_trait::async_trait; use client::GoogleCloudStorageClient; @@ -66,6 +66,13 @@ mod credential; const STORE: &str = "GCS"; +pub fn get_default_capabilities() -> Capabilities { + Capabilities { + // GCS XML API returns results in lexicographic order. + ordered_listing: true, + } +} + /// [`CredentialProvider`] for [`GoogleCloudStorage`] pub type GcpCredentialProvider = Arc>; @@ -77,6 +84,7 @@ pub type GcpSigningCredentialProvider = #[derive(Debug, Clone)] pub struct GoogleCloudStorage { client: Arc, + capabilities: Option, } impl std::fmt::Display for GoogleCloudStorage { @@ -223,6 +231,10 @@ impl ObjectStore for GoogleCloudStorage { self.client.copy_request(from, to, mode).await } + + fn capabilities(&self) -> Capabilities { + self.capabilities.or_else(get_default_capabilities) + } } #[async_trait] diff --git a/src/integration.rs b/src/integration.rs index e68837c5..450d9466 100644 --- a/src/integration.rs +++ b/src/integration.rs @@ -398,6 +398,18 @@ pub async fn put_get_delete_list(storage: &DynObjectStore) { assert_eq!(actual, expected, "{prefix:?} - {offset:?}"); } + if storage.capabilities().ordered_listing { + let actual: Vec<_> = storage + .list(None) + .map_ok(|x| x.location) + .try_collect::>() + .await + .unwrap(); + let mut sorted_files = files.clone(); + sorted_files.sort(); + assert_eq!(actual, sorted_files); + } + // Test bulk delete let paths = vec![ Path::from("a/a.file"), diff --git a/src/lib.rs b/src/lib.rs index d3ea9ee2..89056dff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -742,6 +742,38 @@ pub type MultipartId = String; /// }; /// ``` /// +/// [`Capabilities`]: crate::Capabilities + +/// Optional features supported by an [`ObjectStore`] implementation. +/// +/// Obtain the capabilities of a store by calling [`ObjectStore::capabilities`]. +/// All fields default to `false`; a store sets a field to `true` when it +/// natively supports that feature. +/// +/// The struct is `#[non_exhaustive]` so that new capability flags can be added +/// in future versions without breaking existing code. +/// +/// # Example +/// +/// ``` +/// # use object_store::{ObjectStore, memory::InMemory}; +/// let store = InMemory::new(); +/// let caps = store.capabilities(); +/// if caps.ordered_listing { +/// println!("list() results are in lexicographic order — no need to sort"); +/// } +/// ``` +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct Capabilities { + /// If `true`, [`ObjectStore::list`] and [`ObjectStore::list_with_offset`] + /// return results in ascending lexicographic order by [`Path`]. + /// + /// When this is `true`, callers can rely on the ordering and avoid + /// buffering results solely for the purpose of sorting them. + pub ordered_listing: bool, +} + #[async_trait] pub trait ObjectStore: std::fmt::Display + Send + Sync + Debug + 'static { /// Save the provided `payload` to `location` with the given options @@ -1131,6 +1163,14 @@ pub trait ObjectStore: std::fmt::Display + Send + Sync + Debug + 'static { self.delete(from).await?; Ok(()) } + + /// Return the [`Capabilities`] supported by this store. + /// + /// All capability fields default to `false`. Individual store + /// implementations override this to advertise the features they support. + fn capabilities(&self) -> Capabilities { + Capabilities::default() + } } macro_rules! as_ref_impl { @@ -1202,6 +1242,10 @@ macro_rules! as_ref_impl { ) -> Result<()> { self.as_ref().rename_opts(from, to, options).await } + + fn capabilities(&self) -> Capabilities { + self.as_ref().capabilities() + } } }; } diff --git a/src/memory.rs b/src/memory.rs index 383c553e..08696055 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -29,9 +29,9 @@ use parking_lot::RwLock; use crate::multipart::{MultipartStore, PartId}; use crate::util::InvalidGetRange; use crate::{ - Attributes, GetRange, GetResult, GetResultPayload, ListResult, MultipartId, MultipartUpload, - ObjectMeta, ObjectStore, PutMode, PutMultipartOptions, PutOptions, PutResult, Result, - UpdateVersion, UploadPart, path::Path, + Attributes, Capabilities, GetRange, GetResult, GetResultPayload, ListResult, MultipartId, + MultipartUpload, ObjectMeta, ObjectStore, PutMode, PutMultipartOptions, PutOptions, PutResult, + Result, UpdateVersion, UploadPart, path::Path, }; use crate::{CopyMode, CopyOptions, GetOptions, PutPayload}; @@ -412,6 +412,12 @@ impl ObjectStore for InMemory { Ok(()) } + + fn capabilities(&self) -> Capabilities { + Capabilities { + ordered_listing: true, + } + } } #[async_trait] From 8eb4376b0f4fc5b81eac16d985ca17edc09236f0 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Fri, 22 May 2026 20:44:51 -0700 Subject: [PATCH 02/14] add Capability enum --- src/aws/mod.rs | 14 +++--- src/azure/mod.rs | 15 ++---- src/gcp/builder.rs | 4 +- src/gcp/mod.rs | 11 ++--- src/integration.rs | 7 +-- src/lib.rs | 117 ++++++++++++++++++++++++++++++++------------- src/memory.rs | 10 ++-- 7 files changed, 110 insertions(+), 68 deletions(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 730415be..aa9e06a9 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -43,7 +43,11 @@ use crate::client::list::{ListClient, ListClientExt}; use crate::multipart::{MultipartStore, PartId}; use crate::signer::Signer; use crate::util::STRICT_ENCODE_SET; -use crate::{Capabilities, CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart}; +use crate::{ + Capabilities, Capability, CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult, + MultipartId, MultipartUpload, ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions, + PutOptions, PutPayload, PutResult, Result, UploadPart, +}; static TAGS_HEADER: HeaderName = HeaderName::from_static("x-amz-tagging"); static COPY_SOURCE_HEADER: HeaderName = HeaderName::from_static("x-amz-copy-source"); @@ -75,11 +79,7 @@ use crate::client::parts::Parts; use crate::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; pub use credential::{AwsAuthorizer, AwsCredential}; -pub fn get_default_capabilities() -> Capabilities { - Capabilities { - ordered_listing: false, - } -} +const DEFAULT_CAPABILITIES: Capabilities = Capabilities::new([]); /// Interface for [Amazon S3](https://aws.amazon.com/s3/). #[derive(Debug, Clone)] @@ -418,7 +418,7 @@ impl ObjectStore for AmazonS3 { } fn capabilities(&self) -> Capabilities { - self.capabilities.or_else(get_default_capabilities) + self.capabilities.or(DEFAULT_CAPABILITIES) } } diff --git a/src/azure/mod.rs b/src/azure/mod.rs index 205e5d48..cd55e385 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -24,9 +24,9 @@ //! Unused blocks will automatically be dropped after 7 days. //! use crate::{ - Capabilities, CopyMode, CopyOptions, GetOptions, GetResult, ListResult, MultipartId, - MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, - PutResult, Result, UploadPart, + Capabilities, Capability, CopyMode, CopyOptions, GetOptions, GetResult, ListResult, + MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, + PutPayload, PutResult, Result, UploadPart, multipart::{MultipartStore, PartId}, path::Path, signer::Signer, @@ -57,12 +57,7 @@ pub use builder::{AzureConfigKey, MicrosoftAzureBuilder, split_sas}; pub use credential::AzureCredential; const STORE: &str = "MicrosoftAzure"; - -pub fn get_default_capabilities() -> Capabilities { - Capabilities { - ordered_listing: true, - } -} +const DEFAULT_CAPABILITIES: Capabilities = Capabilities::new([Capability::OrderedListing]); /// Interface for [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/). #[derive(Debug)] @@ -189,7 +184,7 @@ impl ObjectStore for MicrosoftAzure { } fn capabilities(&self) -> Capabilities { - self.capabilities.or_else(get_default_capabilities) + self.capabilities.or(DEFAULT_CAPABILITIES) } } diff --git a/src/gcp/builder.rs b/src/gcp/builder.rs index f24100cb..87821400 100644 --- a/src/gcp/builder.rs +++ b/src/gcp/builder.rs @@ -26,7 +26,9 @@ use crate::gcp::{ GcpCredential, GcpCredentialProvider, GcpSigningCredential, GcpSigningCredentialProvider, GoogleCloudStorage, STORE, credential, }; -use crate::{Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider}; +use crate::{ + Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider, +}; use serde::{Deserialize, Serialize}; use std::str::FromStr; use std::sync::Arc; diff --git a/src/gcp/mod.rs b/src/gcp/mod.rs index 21d5e27e..e9abfab2 100644 --- a/src/gcp/mod.rs +++ b/src/gcp/mod.rs @@ -37,7 +37,6 @@ use std::sync::Arc; use std::time::Duration; -use crate::CopyOptions; use crate::client::CredentialProvider; use crate::gcp::credential::GCSAuthorizer; use crate::signer::Signer; @@ -46,6 +45,7 @@ use crate::{ ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart, multipart::PartId, path::Path, }; +use crate::{Capability, CopyOptions}; use async_trait::async_trait; use client::GoogleCloudStorageClient; use futures_util::stream::{BoxStream, StreamExt}; @@ -66,12 +66,7 @@ mod credential; const STORE: &str = "GCS"; -pub fn get_default_capabilities() -> Capabilities { - Capabilities { - // GCS XML API returns results in lexicographic order. - ordered_listing: true, - } -} +const DEFAULT_CAPABILITIES: Capabilities = Capabilities::new([Capability::OrderedListing]); /// [`CredentialProvider`] for [`GoogleCloudStorage`] pub type GcpCredentialProvider = Arc>; @@ -233,7 +228,7 @@ impl ObjectStore for GoogleCloudStorage { } fn capabilities(&self) -> Capabilities { - self.capabilities.or_else(get_default_capabilities) + self.capabilities.or(DEFAULT_CAPABILITIES) } } diff --git a/src/integration.rs b/src/integration.rs index 450d9466..93a035e4 100644 --- a/src/integration.rs +++ b/src/integration.rs @@ -28,8 +28,9 @@ use crate::list::{PaginatedListOptions, PaginatedListStore}; use crate::multipart::MultipartStore; use crate::path::Path; use crate::{ - Attribute, Attributes, DynObjectStore, Error, GetOptions, GetRange, MultipartUpload, - ObjectStore, ObjectStoreExt, PutMode, PutPayload, UpdateVersion, WriteMultipart, + Attribute, Attributes, Capability, DynObjectStore, Error, GetOptions, GetRange, + MultipartUpload, ObjectStore, ObjectStoreExt, PutMode, PutPayload, UpdateVersion, + WriteMultipart, }; use bytes::Bytes; use futures_util::stream::FuturesUnordered; @@ -398,7 +399,7 @@ pub async fn put_get_delete_list(storage: &DynObjectStore) { assert_eq!(actual, expected, "{prefix:?} - {offset:?}"); } - if storage.capabilities().ordered_listing { + if storage.capabilities().has(Capability::OrderedListing) { let actual: Vec<_> = storage .list(None) .map_ok(|x| x.location) diff --git a/src/lib.rs b/src/lib.rs index 89056dff..a58874c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -571,6 +571,7 @@ pub use client::{ ClientConfigKey, ClientOptions, CredentialProvider, StaticCredentialProvider, backoff::BackoffConfig, retry::RetryConfig, }; +use std::collections::HashSet; #[cfg(all(feature = "cloud", not(target_arch = "wasm32")))] pub use client::Certificate; @@ -742,38 +743,6 @@ pub type MultipartId = String; /// }; /// ``` /// -/// [`Capabilities`]: crate::Capabilities - -/// Optional features supported by an [`ObjectStore`] implementation. -/// -/// Obtain the capabilities of a store by calling [`ObjectStore::capabilities`]. -/// All fields default to `false`; a store sets a field to `true` when it -/// natively supports that feature. -/// -/// The struct is `#[non_exhaustive]` so that new capability flags can be added -/// in future versions without breaking existing code. -/// -/// # Example -/// -/// ``` -/// # use object_store::{ObjectStore, memory::InMemory}; -/// let store = InMemory::new(); -/// let caps = store.capabilities(); -/// if caps.ordered_listing { -/// println!("list() results are in lexicographic order — no need to sort"); -/// } -/// ``` -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -#[non_exhaustive] -pub struct Capabilities { - /// If `true`, [`ObjectStore::list`] and [`ObjectStore::list_with_offset`] - /// return results in ascending lexicographic order by [`Path`]. - /// - /// When this is `true`, callers can rely on the ordering and avoid - /// buffering results solely for the purpose of sorting them. - pub ordered_listing: bool, -} - #[async_trait] pub trait ObjectStore: std::fmt::Display + Send + Sync + Debug + 'static { /// Save the provided `payload` to `location` with the given options @@ -1169,7 +1138,7 @@ pub trait ObjectStore: std::fmt::Display + Send + Sync + Debug + 'static { /// All capability fields default to `false`. Individual store /// implementations override this to advertise the features they support. fn capabilities(&self) -> Capabilities { - Capabilities::default() + Capabilities::new([]) } } @@ -2192,6 +2161,88 @@ impl From for std::io::Error { } } +/// An individual capability that an [`ObjectStore`] implementation may support. +/// +/// Used together with [`Capabilities`] to advertise optional backend features. +/// Obtain the set of supported capabilities via [`ObjectStore::capabilities`]. +/// +/// # String representation +/// +/// Each variant has a stable kebab-case string form accessible via +/// [`Capability::as_str`] and parseable via [`Capability::from_str`]. +/// These strings are intended for configuration, logging, and serialisation. +#[derive(Hash, Eq, PartialEq)] +pub enum Capability { + /// List results from [`ObjectStore::list`] and + /// [`ObjectStore::list_with_offset`] are returned in ascending + /// lexicographic order by [`Path`]. + /// + /// When this capability is present, callers may rely on the ordering and + /// avoid buffering all results solely for sorting purposes. + OrderedListing, +} + +impl Capability { + pub fn as_str(&self) -> &'static str { + match self { + Capability::OrderedListing => "ordered-listing", + } + } + + pub fn from_str(s: &str) -> Option { + match s { + "ordered-listing" => Some(Capability::OrderedListing), + _ => None, + } + } +} + +/// Optional features supported by an [`ObjectStore`] implementation. +/// +/// Obtain the capabilities of a store by calling [`ObjectStore::capabilities`]. +/// All fields default to `false`; a store sets a field to `true` when it +/// natively supports that feature. +/// +/// The struct is `#[non_exhaustive]` so that new capability flags can be added +/// in future versions without breaking existing code. +/// +/// # Example +/// +/// ``` +/// # use object_store::{ObjectStore, memory::InMemory, Capability}; +/// let store = InMemory::new(); +/// if store.capabilities().has(Capability::OrderedListing) { +/// println!("list() results are in lexicographic order — no need to sort"); +/// } +/// ``` +pub struct Capabilities { + supported: HashSet, +} + +impl Capabilities { + /// Create a [`Capabilities`] from an explicit list of supported [`Capability`] values. + /// + /// Any capability not included in `capabilities` is considered unsupported. + /// + /// # Example + /// + /// ``` + /// # use object_store::{Capabilities, Capability}; + /// let caps = Capabilities::new([Capability::OrderedListing]); + /// assert!(caps.has(Capability::OrderedListing)); + /// ``` + pub fn new(capabilities: impl IntoIterator) -> Self { + Self { + supported: capabilities.into_iter().collect(), + } + } + + /// Returns `true` if the given [`Capability`] is supported by this store. + pub fn has(&self, capability: Capability) -> bool { + self.supported.contains(&capability) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/memory.rs b/src/memory.rs index 08696055..be675a85 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -29,9 +29,9 @@ use parking_lot::RwLock; use crate::multipart::{MultipartStore, PartId}; use crate::util::InvalidGetRange; use crate::{ - Attributes, Capabilities, GetRange, GetResult, GetResultPayload, ListResult, MultipartId, - MultipartUpload, ObjectMeta, ObjectStore, PutMode, PutMultipartOptions, PutOptions, PutResult, - Result, UpdateVersion, UploadPart, path::Path, + Attributes, Capabilities, Capability, GetRange, GetResult, GetResultPayload, ListResult, + MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMode, PutMultipartOptions, + PutOptions, PutResult, Result, UpdateVersion, UploadPart, path::Path, }; use crate::{CopyMode, CopyOptions, GetOptions, PutPayload}; @@ -414,9 +414,7 @@ impl ObjectStore for InMemory { } fn capabilities(&self) -> Capabilities { - Capabilities { - ordered_listing: true, - } + Capabilities::new([Capability::OrderedListing]) } } From 54a11e2779de2495444d50fe9bb74b5d279c3175 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Fri, 22 May 2026 21:43:05 -0700 Subject: [PATCH 03/14] move capabilities to src/capabilities.rs --- src/capabilities.rs | 143 ++++++++++++++++++++++++++++++++++++++++++++ src/config.rs | 8 ++- src/integration.rs | 6 +- src/lib.rs | 84 +------------------------- src/memory.rs | 7 ++- 5 files changed, 159 insertions(+), 89 deletions(-) create mode 100644 src/capabilities.rs diff --git a/src/capabilities.rs b/src/capabilities.rs new file mode 100644 index 00000000..08f8dd7e --- /dev/null +++ b/src/capabilities.rs @@ -0,0 +1,143 @@ +use crate::Error; +use std::collections::HashSet; + +/// An individual capability that an [`ObjectStore`] implementation may support. +/// +/// Used together with [`Capabilities`] to advertise optional backend features. +/// Obtain the set of supported capabilities via [`ObjectStore::capabilities`]. +/// +/// # String representation +/// +/// Each variant has a stable kebab-case string form accessible via +/// [`Capability::as_str`] and parseable via [`Capability::from_str`]. +/// These strings are intended for configuration, logging, and serialisation. +#[derive(Hash, Eq, PartialEq, Copy, Clone, Debug)] +pub enum Capability { + /// List results from [`ObjectStore::list`] and + /// [`ObjectStore::list_with_offset`] are returned in ascending + /// lexicographic order by [`Path`]. + /// + /// When this capability is present, callers may rely on the ordering and + /// avoid buffering all results solely for sorting purposes. + OrderedListing, +} + +impl Capability { + /// Returns the stable kebab-case string representation of this capability. + /// + /// The returned string can be round-tripped through [`Capability::from_str`]. + pub fn as_str(&self) -> &'static str { + match self { + Capability::OrderedListing => "ordered-listing", + } + } + + /// Parses a capability from its kebab-case string representation. + /// + /// Returns `None` if `s` does not correspond to any known capability. + pub fn from_str(s: &str) -> Option { + match s { + "ordered-listing" => Some(Capability::OrderedListing), + _ => None, + } + } +} + +/// Optional features supported by an [`ObjectStore`] implementation. +/// +/// Obtain the capabilities of a store by calling [`ObjectStore::capabilities`]. +/// All fields default to `false`; a store sets a field to `true` when it +/// natively supports that feature. +/// +/// The struct is `#[non_exhaustive]` so that new capability flags can be added +/// in future versions without breaking existing code. +/// +/// # Example +/// +/// ``` +/// # use object_store::{ObjectStore, memory::InMemory, Capability}; +/// let store = InMemory::new(); +/// if store.capabilities().has(Capability::OrderedListing) { +/// println!("list() results are in lexicographic order — no need to sort"); +/// } +/// ``` +#[derive(Debug, PartialEq)] +pub struct Capabilities { + supported: HashSet, +} + +impl Capabilities { + /// Create a [`Capabilities`] from an explicit list of supported [`Capability`] values. + /// + /// Any capability not included in `capabilities` is considered unsupported. + /// + /// # Example + /// + /// ``` + /// # use object_store::{Capabilities, Capability}; + /// let caps = Capabilities::new([Capability::OrderedListing]); + /// assert!(caps.has(Capability::OrderedListing)); + /// ``` + pub fn new(capabilities: impl IntoIterator) -> Self { + Self { + supported: capabilities.into_iter().collect(), + } + } + + /// Returns `true` if the given [`Capability`] is supported by this store. + pub fn has(&self, capability: Capability) -> bool { + self.supported.contains(&capability) + } + + pub fn from_str(s: &str) -> crate::Result { + let mut capabilities: Vec = Vec::new(); + for mut cap in s.split(',') { + cap = cap.trim(); + if cap.is_empty() { + continue; + } + match Capability::from_str(cap) { + Some(cap) => capabilities.push(cap), + None => { + return Err(Error::Generic { + store: "Config", + source: format!("invalid capability: {cap}").into(), + }); + } + } + } + Ok(Self::new(capabilities)) + } +} + +#[cfg(test)] +mod tests { + use super::{Capabilities, Capability}; + + #[test] + fn test_capability() { + assert_eq!(Capability::OrderedListing.as_str(), "ordered-listing"); + assert_eq!( + Capability::OrderedListing, + Capability::from_str("ordered-listing").unwrap() + ); + assert_eq!(Capability::from_str("invalid").is_some(), false); + } + + #[test] + fn test_capabilities() { + assert_eq!(Capabilities::from_str("invalid").is_err(), true); + assert_eq!( + Capabilities::from_str("") + .unwrap() + .has(Capability::OrderedListing), + false + ); + assert_eq!( + Capabilities::from_str("ordered-listing") + .unwrap() + .has(Capability::OrderedListing), + true + ); + } +} diff --git a/src/config.rs b/src/config.rs index 29a389d4..6faa8479 100644 --- a/src/config.rs +++ b/src/config.rs @@ -21,7 +21,7 @@ use std::time::Duration; use humantime::{format_duration, parse_duration}; use reqwest::header::HeaderValue; -use crate::{Error, Result}; +use crate::{Capabilities, Capability, Error, Result}; /// Provides deferred parsing of a value /// @@ -121,6 +121,12 @@ impl Parse for HeaderValue { } } +impl Parse for Capabilities { + fn parse(v: &str) -> Result { + Self::from_str(v) + } +} + pub(crate) fn fmt_duration(duration: &ConfigValue) -> String { match duration { ConfigValue::Parsed(v) => format_duration(*v).to_string(), diff --git a/src/integration.rs b/src/integration.rs index 93a035e4..4986bfaa 100644 --- a/src/integration.rs +++ b/src/integration.rs @@ -24,13 +24,13 @@ //! //! They are intended solely for testing purposes. +use crate::capabilities::Capability; use crate::list::{PaginatedListOptions, PaginatedListStore}; use crate::multipart::MultipartStore; use crate::path::Path; use crate::{ - Attribute, Attributes, Capability, DynObjectStore, Error, GetOptions, GetRange, - MultipartUpload, ObjectStore, ObjectStoreExt, PutMode, PutPayload, UpdateVersion, - WriteMultipart, + Attribute, Attributes, DynObjectStore, Error, GetOptions, GetRange, MultipartUpload, + ObjectStore, ObjectStoreExt, PutMode, PutPayload, UpdateVersion, WriteMultipart, }; use bytes::Bytes; use futures_util::stream::FuturesUnordered; diff --git a/src/lib.rs b/src/lib.rs index a58874c9..ade8d455 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -583,6 +583,7 @@ mod tags; pub use tags::TagSet; +pub mod capabilities; pub mod list; pub mod multipart; mod parse; @@ -610,6 +611,7 @@ use crate::path::Path; use crate::util::maybe_spawn_blocking; use async_trait::async_trait; use bytes::Bytes; +pub use capabilities::{Capabilities, Capability}; use chrono::{DateTime, Utc}; use futures_util::{StreamExt, TryStreamExt, stream::BoxStream}; use std::fmt::{Debug, Formatter}; @@ -2161,88 +2163,6 @@ impl From for std::io::Error { } } -/// An individual capability that an [`ObjectStore`] implementation may support. -/// -/// Used together with [`Capabilities`] to advertise optional backend features. -/// Obtain the set of supported capabilities via [`ObjectStore::capabilities`]. -/// -/// # String representation -/// -/// Each variant has a stable kebab-case string form accessible via -/// [`Capability::as_str`] and parseable via [`Capability::from_str`]. -/// These strings are intended for configuration, logging, and serialisation. -#[derive(Hash, Eq, PartialEq)] -pub enum Capability { - /// List results from [`ObjectStore::list`] and - /// [`ObjectStore::list_with_offset`] are returned in ascending - /// lexicographic order by [`Path`]. - /// - /// When this capability is present, callers may rely on the ordering and - /// avoid buffering all results solely for sorting purposes. - OrderedListing, -} - -impl Capability { - pub fn as_str(&self) -> &'static str { - match self { - Capability::OrderedListing => "ordered-listing", - } - } - - pub fn from_str(s: &str) -> Option { - match s { - "ordered-listing" => Some(Capability::OrderedListing), - _ => None, - } - } -} - -/// Optional features supported by an [`ObjectStore`] implementation. -/// -/// Obtain the capabilities of a store by calling [`ObjectStore::capabilities`]. -/// All fields default to `false`; a store sets a field to `true` when it -/// natively supports that feature. -/// -/// The struct is `#[non_exhaustive]` so that new capability flags can be added -/// in future versions without breaking existing code. -/// -/// # Example -/// -/// ``` -/// # use object_store::{ObjectStore, memory::InMemory, Capability}; -/// let store = InMemory::new(); -/// if store.capabilities().has(Capability::OrderedListing) { -/// println!("list() results are in lexicographic order — no need to sort"); -/// } -/// ``` -pub struct Capabilities { - supported: HashSet, -} - -impl Capabilities { - /// Create a [`Capabilities`] from an explicit list of supported [`Capability`] values. - /// - /// Any capability not included in `capabilities` is considered unsupported. - /// - /// # Example - /// - /// ``` - /// # use object_store::{Capabilities, Capability}; - /// let caps = Capabilities::new([Capability::OrderedListing]); - /// assert!(caps.has(Capability::OrderedListing)); - /// ``` - pub fn new(capabilities: impl IntoIterator) -> Self { - Self { - supported: capabilities.into_iter().collect(), - } - } - - /// Returns `true` if the given [`Capability`] is supported by this store. - pub fn has(&self, capability: Capability) -> bool { - self.supported.contains(&capability) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/memory.rs b/src/memory.rs index be675a85..c8fed468 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -26,12 +26,13 @@ use chrono::{DateTime, Utc}; use futures_util::{StreamExt, stream::BoxStream}; use parking_lot::RwLock; +use crate::capabilities::Capability; use crate::multipart::{MultipartStore, PartId}; use crate::util::InvalidGetRange; use crate::{ - Attributes, Capabilities, Capability, GetRange, GetResult, GetResultPayload, ListResult, - MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMode, PutMultipartOptions, - PutOptions, PutResult, Result, UpdateVersion, UploadPart, path::Path, + Attributes, Capabilities, GetRange, GetResult, GetResultPayload, ListResult, MultipartId, + MultipartUpload, ObjectMeta, ObjectStore, PutMode, PutMultipartOptions, PutOptions, PutResult, + Result, UpdateVersion, UploadPart, path::Path, }; use crate::{CopyMode, CopyOptions, GetOptions, PutPayload}; From 585554752b58ca470aed3f726535aec755dc7959 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Mon, 25 May 2026 13:21:36 -0700 Subject: [PATCH 04/14] update aws builder config --- src/aws/builder.rs | 54 ++++++++++++++++++++++++++++++++++++++++---- src/aws/mod.rs | 13 +++++++---- src/azure/builder.rs | 2 +- src/azure/mod.rs | 8 +++++-- src/capabilities.rs | 25 ++++++++++++++------ src/config.rs | 2 +- src/gcp/builder.rs | 1 + src/gcp/mod.rs | 6 +++-- src/lib.rs | 1 - 9 files changed, 89 insertions(+), 23 deletions(-) diff --git a/src/aws/builder.rs b/src/aws/builder.rs index 7aa50df0..440d892e 100644 --- a/src/aws/builder.rs +++ b/src/aws/builder.rs @@ -196,7 +196,7 @@ pub struct AmazonS3Builder { /// The [`HttpConnector`] to use http_connector: Option>, /// Capabilities to advertise for this store instance - capabilities: Option, + capabilities: Option>, } /// Configuration keys for [`AmazonS3Builder`] @@ -467,6 +467,9 @@ pub enum AmazonS3ConfigKey { /// Encryption options Encryption(S3EncryptionConfigKey), + + /// Override the capabilities advertised by this store. + Capabilities, } impl AsRef for AmazonS3ConfigKey { @@ -501,6 +504,7 @@ impl AsRef for AmazonS3ConfigKey { Self::RequestPayer => "aws_request_payer", Self::Client(opt) => opt.as_ref(), Self::Encryption(opt) => opt.as_ref(), + Self::Capabilities => "aws_capabilities", } } } @@ -561,6 +565,7 @@ impl FromStr for AmazonS3ConfigKey { "aws_sse_customer_key_base64" | "sse_customer_key_base64" => Ok(Self::Encryption( S3EncryptionConfigKey::CustomerEncryptionKey, )), + "aws_capabilities" => Ok(Self::Capabilities), _ => match s.strip_prefix("aws_").unwrap_or(s).parse() { Ok(key) => Ok(Self::Client(key)), Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), @@ -713,6 +718,9 @@ impl AmazonS3Builder { self.encryption_customer_key_base64 = Some(value.into()) } }, + AmazonS3ConfigKey::Capabilities => { + self.capabilities = Some(ConfigValue::Deferred(value.into())) + } }; self } @@ -769,6 +777,7 @@ impl AmazonS3Builder { AmazonS3ConfigKey::DisableTagging => Some(self.disable_tagging.to_string()), AmazonS3ConfigKey::DisableBulkDelete => Some(self.disable_bulk_delete.to_string()), AmazonS3ConfigKey::RequestPayer => Some(self.request_payer.to_string()), + AmazonS3ConfigKey::Capabilities => self.capabilities.as_ref().map(ToString::to_string), AmazonS3ConfigKey::Encryption(key) => match key { S3EncryptionConfigKey::ServerSideEncryption => { self.encryption_type.as_ref().map(ToString::to_string) @@ -1116,7 +1125,7 @@ impl AmazonS3Builder { /// method if you are connecting to an S3-compatible endpoint whose /// behaviour differs from the standard S3 API. pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { - self.capabilities = Some(capabilities); + self.capabilities = Some(ConfigValue::Parsed(capabilities)); self } @@ -1303,7 +1312,7 @@ impl AmazonS3Builder { Ok(AmazonS3 { client, - capabilities: self.capabilities, + capabilities: self.capabilities.map(|x| x.get()).transpose()?, }) } } @@ -1553,6 +1562,7 @@ impl From for HeaderMap { #[cfg(test)] mod tests { use super::*; + use crate::Capability; use std::collections::HashMap; #[test] @@ -1570,6 +1580,7 @@ mod tests { ("aws_session_token", aws_session_token.clone()), ("aws_unsigned_payload", "true".to_string()), ("aws_checksum_algorithm", "sha256".to_string()), + ("aws_capabilities", "ordered-listing".to_string()), ]); let builder = options @@ -1589,6 +1600,14 @@ mod tests { Checksum::SHA256 ); assert!(builder.unsigned_payload.get().unwrap()); + assert!( + builder + .capabilities + .unwrap() + .get() + .unwrap() + .has(Capability::OrderedListing) + ); } #[test] @@ -1643,7 +1662,8 @@ mod tests { .with_config( "aws_sse_customer_key_base64".parse().unwrap(), "some_customer_key", - ); + ) + .with_config(AmazonS3ConfigKey::Capabilities, "ordered-listing"); assert_eq!( builder @@ -1703,6 +1723,12 @@ mod tests { .unwrap(), "some_customer_key" ); + assert_eq!( + builder + .get_config_value(&"aws_capabilities".parse().unwrap()) + .unwrap(), + "ordered-listing" + ); } #[test] @@ -1926,6 +1952,26 @@ mod tests { assert!(s3.client.config.request_payer); } + #[test] + fn test_parse_capabilities() { + // Default: ordered listing disabled + let s3 = AmazonS3Builder::new() + .with_bucket_name("bucket") + .with_region("region") + .build() + .unwrap(); + assert!(!s3.capabilities.is_some()); + + // Explicit override via with_capabilities: no capabilities + let s3 = AmazonS3Builder::new() + .with_capabilities(Capabilities::new([Capability::OrderedListing])) + .with_bucket_name("bucket") + .with_region("region") + .build() + .unwrap(); + assert!(s3.capabilities.unwrap().has(Capability::OrderedListing)); + } + #[test] fn test_parse_bucket_az() { let cases = [ diff --git a/src/aws/mod.rs b/src/aws/mod.rs index aa9e06a9..079dca7d 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -44,9 +44,9 @@ use crate::multipart::{MultipartStore, PartId}; use crate::signer::Signer; use crate::util::STRICT_ENCODE_SET; use crate::{ - Capabilities, Capability, CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult, - MultipartId, MultipartUpload, ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions, - PutOptions, PutPayload, PutResult, Result, UploadPart, + Capabilities, CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartId, + MultipartUpload, ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions, PutOptions, + PutPayload, PutResult, Result, UploadPart, }; static TAGS_HEADER: HeaderName = HeaderName::from_static("x-amz-tagging"); @@ -79,7 +79,9 @@ use crate::client::parts::Parts; use crate::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; pub use credential::{AwsAuthorizer, AwsCredential}; -const DEFAULT_CAPABILITIES: Capabilities = Capabilities::new([]); +fn get_default_capabilities() -> Capabilities { + return Capabilities::new([]); +} /// Interface for [Amazon S3](https://aws.amazon.com/s3/). #[derive(Debug, Clone)] @@ -418,7 +420,7 @@ impl ObjectStore for AmazonS3 { } fn capabilities(&self) -> Capabilities { - self.capabilities.or(DEFAULT_CAPABILITIES) + self.capabilities.clone().unwrap_or_else(get_default_capabilities) } } @@ -714,6 +716,7 @@ mod tests { tagging( Arc::new(AmazonS3 { client: Arc::clone(&integration.client), + capabilities: None, }), !config.disable_tagging, |p| { diff --git a/src/azure/builder.rs b/src/azure/builder.rs index a5262807..7f80d887 100644 --- a/src/azure/builder.rs +++ b/src/azure/builder.rs @@ -917,7 +917,7 @@ impl MicrosoftAzureBuilder { /// you are connecting to an endpoint whose behaviour differs from the /// standard Azure Blob Storage API. pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { - self.capabilities = capabilities; + self.capabilities = Some(capabilities); self } diff --git a/src/azure/mod.rs b/src/azure/mod.rs index cd55e385..9fafa1a9 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -57,7 +57,10 @@ pub use builder::{AzureConfigKey, MicrosoftAzureBuilder, split_sas}; pub use credential::AzureCredential; const STORE: &str = "MicrosoftAzure"; -const DEFAULT_CAPABILITIES: Capabilities = Capabilities::new([Capability::OrderedListing]); + +fn get_default_capabilities() -> Capabilities { + Capabilities::new([Capability::OrderedListing]) +} /// Interface for [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/). #[derive(Debug)] @@ -184,7 +187,7 @@ impl ObjectStore for MicrosoftAzure { } fn capabilities(&self) -> Capabilities { - self.capabilities.or(DEFAULT_CAPABILITIES) + self.capabilities.clone().unwrap_or_else(get_default_capabilities) } } @@ -369,6 +372,7 @@ mod tests { tagging( Arc::new(MicrosoftAzure { client: Arc::clone(&integration.client), + capabilities: None, }), validate, |p| { diff --git a/src/capabilities.rs b/src/capabilities.rs index 08f8dd7e..1a071aeb 100644 --- a/src/capabilities.rs +++ b/src/capabilities.rs @@ -1,3 +1,6 @@ +//! Capability advertisement for [`ObjectStore`](crate::ObjectStore) implementations. +//! +//! See [`Capabilities`] and [`Capability`] for details. use crate::Error; use std::collections::HashSet; @@ -5,12 +8,6 @@ use std::collections::HashSet; /// /// Used together with [`Capabilities`] to advertise optional backend features. /// Obtain the set of supported capabilities via [`ObjectStore::capabilities`]. -/// -/// # String representation -/// -/// Each variant has a stable kebab-case string form accessible via -/// [`Capability::as_str`] and parseable via [`Capability::from_str`]. -/// These strings are intended for configuration, logging, and serialisation. #[derive(Hash, Eq, PartialEq, Copy, Clone, Debug)] pub enum Capability { /// List results from [`ObjectStore::list`] and @@ -61,7 +58,7 @@ impl Capability { /// println!("list() results are in lexicographic order — no need to sort"); /// } /// ``` -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Default)] pub struct Capabilities { supported: HashSet, } @@ -89,6 +86,7 @@ impl Capabilities { self.supported.contains(&capability) } + /// Parses a comma-separated list of capability names into a [`Capabilities`]. pub fn from_str(s: &str) -> crate::Result { let mut capabilities: Vec = Vec::new(); for mut cap in s.split(',') { @@ -110,6 +108,19 @@ impl Capabilities { } } +impl std::fmt::Display for Capabilities { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut iter = self.supported.iter(); + if let Some(cap) = iter.next() { + write!(f, "{}", cap.as_str())?; + } + for cap in iter { + write!(f, ", {}", cap.as_str())?; + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::{Capabilities, Capability}; diff --git a/src/config.rs b/src/config.rs index 6faa8479..871248f9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -21,7 +21,7 @@ use std::time::Duration; use humantime::{format_duration, parse_duration}; use reqwest::header::HeaderValue; -use crate::{Capabilities, Capability, Error, Result}; +use crate::{Capabilities, Error, Result}; /// Provides deferred parsing of a value /// diff --git a/src/gcp/builder.rs b/src/gcp/builder.rs index 87821400..3f1ae4ea 100644 --- a/src/gcp/builder.rs +++ b/src/gcp/builder.rs @@ -261,6 +261,7 @@ impl Default for GoogleCloudStorageBuilder { skip_signature: Default::default(), signing_credentials: None, http_connector: None, + capabilities: None, } } } diff --git a/src/gcp/mod.rs b/src/gcp/mod.rs index e9abfab2..6e4c1fad 100644 --- a/src/gcp/mod.rs +++ b/src/gcp/mod.rs @@ -66,7 +66,9 @@ mod credential; const STORE: &str = "GCS"; -const DEFAULT_CAPABILITIES: Capabilities = Capabilities::new([Capability::OrderedListing]); +fn get_default_capabilities() -> Capabilities { + Capabilities::new([Capability::OrderedListing]) +} /// [`CredentialProvider`] for [`GoogleCloudStorage`] pub type GcpCredentialProvider = Arc>; @@ -228,7 +230,7 @@ impl ObjectStore for GoogleCloudStorage { } fn capabilities(&self) -> Capabilities { - self.capabilities.or(DEFAULT_CAPABILITIES) + self.capabilities.clone().unwrap_or_else(get_default_capabilities) } } diff --git a/src/lib.rs b/src/lib.rs index ade8d455..227cc5e7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -571,7 +571,6 @@ pub use client::{ ClientConfigKey, ClientOptions, CredentialProvider, StaticCredentialProvider, backoff::BackoffConfig, retry::RetryConfig, }; -use std::collections::HashSet; #[cfg(all(feature = "cloud", not(target_arch = "wasm32")))] pub use client::Certificate; From 28c579bf696a5ea34d3cd1c18a14022762b30cfd Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Tue, 26 May 2026 10:13:55 -0700 Subject: [PATCH 05/14] cargo fmt --- src/aws/mod.rs | 4 +++- src/azure/mod.rs | 4 +++- src/gcp/mod.rs | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 079dca7d..9df7c509 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -420,7 +420,9 @@ impl ObjectStore for AmazonS3 { } fn capabilities(&self) -> Capabilities { - self.capabilities.clone().unwrap_or_else(get_default_capabilities) + self.capabilities + .clone() + .unwrap_or_else(get_default_capabilities) } } diff --git a/src/azure/mod.rs b/src/azure/mod.rs index 9fafa1a9..decec93d 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -187,7 +187,9 @@ impl ObjectStore for MicrosoftAzure { } fn capabilities(&self) -> Capabilities { - self.capabilities.clone().unwrap_or_else(get_default_capabilities) + self.capabilities + .clone() + .unwrap_or_else(get_default_capabilities) } } diff --git a/src/gcp/mod.rs b/src/gcp/mod.rs index 6e4c1fad..650a2012 100644 --- a/src/gcp/mod.rs +++ b/src/gcp/mod.rs @@ -230,7 +230,9 @@ impl ObjectStore for GoogleCloudStorage { } fn capabilities(&self) -> Capabilities { - self.capabilities.clone().unwrap_or_else(get_default_capabilities) + self.capabilities + .clone() + .unwrap_or_else(get_default_capabilities) } } From 519eb8e9c990835c48a98d70affae08e54e63a64 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Tue, 26 May 2026 11:15:13 -0700 Subject: [PATCH 06/14] fix clippy --- src/capabilities.rs | 86 ++++++++++++++++++++++++++------------------- src/chunked.rs | 9 ++--- src/limit.rs | 10 +++--- src/prefix.rs | 9 ++--- src/throttle.rs | 6 +++- 5 files changed, 69 insertions(+), 51 deletions(-) diff --git a/src/capabilities.rs b/src/capabilities.rs index 1a071aeb..f6085670 100644 --- a/src/capabilities.rs +++ b/src/capabilities.rs @@ -1,9 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + //! Capability advertisement for [`ObjectStore`](crate::ObjectStore) implementations. //! //! See [`Capabilities`] and [`Capability`] for details. use crate::Error; use std::collections::HashSet; +const ORDERED_LISTING: &str = "ordered-listing"; + /// An individual capability that an [`ObjectStore`] implementation may support. /// /// Used together with [`Capabilities`] to advertise optional backend features. @@ -19,23 +38,27 @@ pub enum Capability { OrderedListing, } -impl Capability { - /// Returns the stable kebab-case string representation of this capability. - /// - /// The returned string can be round-tripped through [`Capability::from_str`]. - pub fn as_str(&self) -> &'static str { - match self { - Capability::OrderedListing => "ordered-listing", - } - } +impl std::str::FromStr for Capability { + type Err = Error; /// Parses a capability from its kebab-case string representation. /// /// Returns `None` if `s` does not correspond to any known capability. - pub fn from_str(s: &str) -> Option { + fn from_str(s: &str) -> Result { match s { - "ordered-listing" => Some(Capability::OrderedListing), - _ => None, + ORDERED_LISTING => Ok(Self::OrderedListing), + cap => Err(Error::Generic { + store: "Config", + source: format!("invalid capability: {cap}").into(), + }), + } + } +} + +impl std::fmt::Display for Capability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::OrderedListing => write!(f, "{}", ORDERED_LISTING), } } } @@ -85,24 +108,20 @@ impl Capabilities { pub fn has(&self, capability: Capability) -> bool { self.supported.contains(&capability) } +} + +impl std::str::FromStr for Capabilities { + type Err = Error; /// Parses a comma-separated list of capability names into a [`Capabilities`]. - pub fn from_str(s: &str) -> crate::Result { + fn from_str(s: &str) -> crate::Result { let mut capabilities: Vec = Vec::new(); for mut cap in s.split(',') { cap = cap.trim(); if cap.is_empty() { continue; } - match Capability::from_str(cap) { - Some(cap) => capabilities.push(cap), - None => { - return Err(Error::Generic { - store: "Config", - source: format!("invalid capability: {cap}").into(), - }); - } - } + capabilities.push(cap.parse::()?); } Ok(Self::new(capabilities)) } @@ -112,10 +131,10 @@ impl std::fmt::Display for Capabilities { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut iter = self.supported.iter(); if let Some(cap) = iter.next() { - write!(f, "{}", cap.as_str())?; + write!(f, "{}", cap)?; } for cap in iter { - write!(f, ", {}", cap.as_str())?; + write!(f, ", {}", cap)?; } Ok(()) } @@ -127,27 +146,20 @@ mod tests { #[test] fn test_capability() { - assert_eq!(Capability::OrderedListing.as_str(), "ordered-listing"); + assert_eq!(format!("{}", Capability::OrderedListing), "ordered-listing"); assert_eq!( Capability::OrderedListing, - Capability::from_str("ordered-listing").unwrap() + "ordered-listing".parse::().unwrap() ); - assert_eq!(Capability::from_str("invalid").is_some(), false); + assert_eq!("invalid".parse::().is_ok(), false); } #[test] fn test_capabilities() { - assert_eq!(Capabilities::from_str("invalid").is_err(), true); - assert_eq!( - Capabilities::from_str("") - .unwrap() - .has(Capability::OrderedListing), - false - ); + assert_eq!("invalid".parse::().is_err(), true); + assert_eq!("".parse::().unwrap().has(Capability::OrderedListing), false); assert_eq!( - Capabilities::from_str("ordered-listing") - .unwrap() - .has(Capability::OrderedListing), + "ordered-listing".parse::().unwrap().has(Capability::OrderedListing), true ); } diff --git a/src/chunked.rs b/src/chunked.rs index 870540a2..8e1cfb31 100644 --- a/src/chunked.rs +++ b/src/chunked.rs @@ -27,10 +27,7 @@ use futures_util::StreamExt; use futures_util::stream::BoxStream; use crate::path::Path; -use crate::{ - CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, - ObjectStore, PutMultipartOptions, PutOptions, PutResult, RenameOptions, -}; +use crate::{Capabilities, CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutResult, RenameOptions}; use crate::{PutPayload, Result}; /// Wraps a [`ObjectStore`] and makes its get response return chunks @@ -169,6 +166,10 @@ impl ObjectStore for ChunkedStore { async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> { self.inner.rename_opts(from, to, options).await } + + fn capabilities(&self) -> Capabilities { + self.inner.capabilities() + } } #[cfg(test)] diff --git a/src/limit.rs b/src/limit.rs index fa29d1b2..2b302df7 100644 --- a/src/limit.rs +++ b/src/limit.rs @@ -17,11 +17,7 @@ //! An object store that limits the maximum concurrency of the wrapped implementation -use crate::{ - BoxStream, CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, - ObjectMeta, ObjectStore, Path, PutMultipartOptions, PutOptions, PutPayload, PutResult, - RenameOptions, Result, StreamExt, UploadPart, -}; +use crate::{BoxStream, Capabilities, CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, Path, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result, StreamExt, UploadPart}; use async_trait::async_trait; use bytes::Bytes; use futures_util::{FutureExt, Stream}; @@ -162,6 +158,10 @@ impl ObjectStore for LimitStore { let _permit = self.semaphore.acquire().await.unwrap(); self.inner.rename_opts(from, to, options).await } + + fn capabilities(&self) -> Capabilities { + self.inner.capabilities() + } } fn permit_get_result(r: GetResult, permit: OwnedSemaphorePermit) -> GetResult { diff --git a/src/prefix.rs b/src/prefix.rs index 46cd6816..abadc2b1 100644 --- a/src/prefix.rs +++ b/src/prefix.rs @@ -24,10 +24,7 @@ use crate::multipart::{MultipartStore, PartId}; use crate::path::Path; #[cfg(feature = "cloud")] use crate::signer::Signer; -use crate::{ - CopyOptions, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, - ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result, -}; +use crate::{Capabilities, CopyOptions, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result}; /// Store wrapper that applies a constant prefix to all paths handled by the store. #[derive(Debug, Clone)] @@ -200,6 +197,10 @@ impl ObjectStore for PrefixStore { let full_to = self.full_path(to); self.inner.rename_opts(&full_from, &full_to, options).await } + + fn capabilities(&self) -> Capabilities { + self.inner.capabilities() + } } #[async_trait::async_trait] diff --git a/src/throttle.rs b/src/throttle.rs index 695afe40..e6378265 100644 --- a/src/throttle.rs +++ b/src/throttle.rs @@ -21,7 +21,7 @@ use std::ops::Range; use std::{convert::TryInto, sync::Arc}; use crate::multipart::{MultipartStore, PartId}; -use crate::{CopyOptions, GetOptions, RenameOptions, UploadPart}; +use crate::{Capabilities, CopyOptions, GetOptions, RenameOptions, UploadPart}; use crate::{ GetResult, GetResultPayload, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, path::Path, @@ -263,6 +263,10 @@ impl ObjectStore for ThrottledStore { self.inner.rename_opts(from, to, options).await } + + fn capabilities(&self) -> Capabilities { + self.inner.capabilities() + } } /// Saturated `usize` to `u32` cast. From 306bc0417084f43724db58da03df2243b2b56a49 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Tue, 26 May 2026 13:47:06 -0700 Subject: [PATCH 07/14] cargo fmt --- src/capabilities.rs | 12 ++++++++++-- src/chunked.rs | 6 +++++- src/limit.rs | 6 +++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/capabilities.rs b/src/capabilities.rs index f6085670..f572e3c8 100644 --- a/src/capabilities.rs +++ b/src/capabilities.rs @@ -157,9 +157,17 @@ mod tests { #[test] fn test_capabilities() { assert_eq!("invalid".parse::().is_err(), true); - assert_eq!("".parse::().unwrap().has(Capability::OrderedListing), false); assert_eq!( - "ordered-listing".parse::().unwrap().has(Capability::OrderedListing), + "".parse::() + .unwrap() + .has(Capability::OrderedListing), + false + ); + assert_eq!( + "ordered-listing" + .parse::() + .unwrap() + .has(Capability::OrderedListing), true ); } diff --git a/src/chunked.rs b/src/chunked.rs index 8e1cfb31..58a026e4 100644 --- a/src/chunked.rs +++ b/src/chunked.rs @@ -27,7 +27,11 @@ use futures_util::StreamExt; use futures_util::stream::BoxStream; use crate::path::Path; -use crate::{Capabilities, CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutResult, RenameOptions}; +use crate::{ + Capabilities, CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, + MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutResult, + RenameOptions, +}; use crate::{PutPayload, Result}; /// Wraps a [`ObjectStore`] and makes its get response return chunks diff --git a/src/limit.rs b/src/limit.rs index 2b302df7..ae099ca2 100644 --- a/src/limit.rs +++ b/src/limit.rs @@ -17,7 +17,11 @@ //! An object store that limits the maximum concurrency of the wrapped implementation -use crate::{BoxStream, Capabilities, CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, Path, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result, StreamExt, UploadPart}; +use crate::{ + BoxStream, Capabilities, CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, + MultipartUpload, ObjectMeta, ObjectStore, Path, PutMultipartOptions, PutOptions, PutPayload, + PutResult, RenameOptions, Result, StreamExt, UploadPart, +}; use async_trait::async_trait; use bytes::Bytes; use futures_util::{FutureExt, Stream}; From cf82b300527e39d88426edca047d26d955fb82f3 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Tue, 9 Jun 2026 20:04:26 -0700 Subject: [PATCH 08/14] wip --- src/aws/builder.rs | 5 ----- src/aws/mod.rs | 1 + src/azure/builder.rs | 5 ----- src/azure/mod.rs | 3 ++- src/capabilities.rs | 3 ++- src/gcp/builder.rs | 5 ----- src/gcp/mod.rs | 1 + src/integration.rs | 13 +++++++++++-- src/lib.rs | 2 +- 9 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/aws/builder.rs b/src/aws/builder.rs index 440d892e..e85a74ee 100644 --- a/src/aws/builder.rs +++ b/src/aws/builder.rs @@ -1119,11 +1119,6 @@ impl AmazonS3Builder { } /// Override the [`Capabilities`] advertised by this store. - /// - /// By default the store reports `ordered_listing: true` because S3 - /// `ListObjectsV2` returns results in lexicographic order. Use this - /// method if you are connecting to an S3-compatible endpoint whose - /// behaviour differs from the standard S3 API. pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { self.capabilities = Some(ConfigValue::Parsed(capabilities)); self diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 9df7c509..520d2a38 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -79,6 +79,7 @@ use crate::client::parts::Parts; use crate::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; pub use credential::{AwsAuthorizer, AwsCredential}; +/// OrderedListing capability depends on the bucket type, it's not enabled for directory bucket. fn get_default_capabilities() -> Capabilities { return Capabilities::new([]); } diff --git a/src/azure/builder.rs b/src/azure/builder.rs index 7f80d887..b16a22f0 100644 --- a/src/azure/builder.rs +++ b/src/azure/builder.rs @@ -911,11 +911,6 @@ impl MicrosoftAzureBuilder { } /// Override the [`Capabilities`] advertised by this store. - /// - /// By default the store reports `ordered_listing: true` because Azure Blob - /// Storage returns list results in lexicographic order. Use this method if - /// you are connecting to an endpoint whose behaviour differs from the - /// standard Azure Blob Storage API. pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { self.capabilities = Some(capabilities); self diff --git a/src/azure/mod.rs b/src/azure/mod.rs index decec93d..1b0d3881 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -58,8 +58,9 @@ pub use credential::AzureCredential; const STORE: &str = "MicrosoftAzure"; +// OrderedListing capability is not supported by with Azure Storage Hierarchical Namespace is enabled. fn get_default_capabilities() -> Capabilities { - Capabilities::new([Capability::OrderedListing]) + Capabilities::new([]) } /// Interface for [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/). diff --git a/src/capabilities.rs b/src/capabilities.rs index f572e3c8..2d25a053 100644 --- a/src/capabilities.rs +++ b/src/capabilities.rs @@ -26,8 +26,9 @@ const ORDERED_LISTING: &str = "ordered-listing"; /// An individual capability that an [`ObjectStore`] implementation may support. /// /// Used together with [`Capabilities`] to advertise optional backend features. -/// Obtain the set of supported capabilities via [`ObjectStore::capabilities`]. +/// Get the set of supported capabilities via [`ObjectStore::capabilities`]. #[derive(Hash, Eq, PartialEq, Copy, Clone, Debug)] +#[non_exhaustive] pub enum Capability { /// List results from [`ObjectStore::list`] and /// [`ObjectStore::list_with_offset`] are returned in ascending diff --git a/src/gcp/builder.rs b/src/gcp/builder.rs index 3f1ae4ea..a36da995 100644 --- a/src/gcp/builder.rs +++ b/src/gcp/builder.rs @@ -540,11 +540,6 @@ impl GoogleCloudStorageBuilder { } /// Override the [`Capabilities`] advertised by this store. - /// - /// By default the store reports `ordered_listing: true` because GCS - /// returns list results in lexicographic order. Use this method if you - /// are connecting to an endpoint whose behaviour differs from the - /// standard GCS API. pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { self.capabilities = Some(capabilities); self diff --git a/src/gcp/mod.rs b/src/gcp/mod.rs index 650a2012..74ddcec9 100644 --- a/src/gcp/mod.rs +++ b/src/gcp/mod.rs @@ -66,6 +66,7 @@ mod credential; const STORE: &str = "GCS"; +// OrderedListing is supported by all GCP bucket types. fn get_default_capabilities() -> Capabilities { Capabilities::new([Capability::OrderedListing]) } diff --git a/src/integration.rs b/src/integration.rs index 4986bfaa..6949a69c 100644 --- a/src/integration.rs +++ b/src/integration.rs @@ -400,15 +400,24 @@ pub async fn put_get_delete_list(storage: &DynObjectStore) { } if storage.capabilities().has(Capability::OrderedListing) { + let mut sorted_files = files.clone(); + sorted_files.sort(); + let actual: Vec<_> = storage .list(None) .map_ok(|x| x.location) .try_collect::>() .await .unwrap(); - let mut sorted_files = files.clone(); - sorted_files.sort(); assert_eq!(actual, sorted_files); + + let actual: Vec<_> = storage + .list_with_offset(None, &sorted_files[1]) + .map_ok(|x| x.location) + .try_collect::>() + .await + .unwrap(); + assert_eq!(actual, sorted_files[2..]); } // Test bulk delete diff --git a/src/lib.rs b/src/lib.rs index 227cc5e7..4676e667 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1136,7 +1136,7 @@ pub trait ObjectStore: std::fmt::Display + Send + Sync + Debug + 'static { /// Return the [`Capabilities`] supported by this store. /// - /// All capability fields default to `false`. Individual store + /// By default, an empty set of capabilities is returned. Individual store /// implementations override this to advertise the features they support. fn capabilities(&self) -> Capabilities { Capabilities::new([]) From bd53358497c560620d717ff3c515f771dc998923 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Tue, 9 Jun 2026 20:07:23 -0700 Subject: [PATCH 09/14] update comments --- src/capabilities.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/capabilities.rs b/src/capabilities.rs index 2d25a053..b23f0985 100644 --- a/src/capabilities.rs +++ b/src/capabilities.rs @@ -66,9 +66,8 @@ impl std::fmt::Display for Capability { /// Optional features supported by an [`ObjectStore`] implementation. /// -/// Obtain the capabilities of a store by calling [`ObjectStore::capabilities`]. -/// All fields default to `false`; a store sets a field to `true` when it -/// natively supports that feature. +/// Get the capabilities of a store by calling [`ObjectStore::capabilities`]. +/// Check whether [`Capability`] is supported by calling [`Capabilities::has`] method. /// /// The struct is `#[non_exhaustive]` so that new capability flags can be added /// in future versions without breaking existing code. From eadba2bb38b8b67de84654108daff4f200a74f52 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Tue, 9 Jun 2026 21:20:13 -0700 Subject: [PATCH 10/14] add capabilities config for azure and gcp --- src/azure/builder.rs | 19 +++++++++++++------ src/gcp/builder.rs | 17 +++++++++++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/azure/builder.rs b/src/azure/builder.rs index b16a22f0..e8d5b17f 100644 --- a/src/azure/builder.rs +++ b/src/azure/builder.rs @@ -23,9 +23,7 @@ use crate::azure::credential::{ use crate::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzure, STORE}; use crate::client::{HttpConnector, TokenCredentialProvider, http_connector}; use crate::config::ConfigValue; -use crate::{ - Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider, -}; +use crate::{Capabilities, ClientConfigKey, ClientOptions, ObjectStoreExt, Result, RetryConfig, StaticCredentialProvider}; use percent_encoding::percent_decode_str; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -183,7 +181,7 @@ pub struct MicrosoftAzureBuilder { /// The [`HttpConnector`] to use http_connector: Option>, /// Capabilities to advertise for this store instance - capabilities: Option, + capabilities: Option>, } /// Configuration keys for [`MicrosoftAzureBuilder`] @@ -386,6 +384,9 @@ pub enum AzureConfigKey { /// Client options Client(ClientConfigKey), + + /// Override the capabilities advertised by this store. + Capabilities, } impl AsRef for AzureConfigKey { @@ -415,6 +416,7 @@ impl AsRef for AzureConfigKey { Self::FabricSessionToken => "azure_fabric_session_token", Self::FabricClusterIdentifier => "azure_fabric_cluster_identifier", Self::Client(key) => key.as_ref(), + Self::Capabilities => "azure_capabilities", } } } @@ -472,6 +474,7 @@ impl FromStr for AzureConfigKey { } // Backwards compatibility "azure_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)), + "azure_capabilities" => Ok(Self::Capabilities), _ => match s.strip_prefix("azure_").unwrap_or(s).parse() { Ok(key) => Ok(Self::Client(key)), Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), @@ -598,6 +601,9 @@ impl MicrosoftAzureBuilder { AzureConfigKey::FabricClusterIdentifier => { self.fabric_cluster_identifier = Some(value.into()) } + AzureConfigKey::Capabilities => { + self.capabilities = Some(ConfigValue::Deferred(value.into())) + } }; self } @@ -639,6 +645,7 @@ impl MicrosoftAzureBuilder { AzureConfigKey::FabricWorkloadHost => self.fabric_workload_host.clone(), AzureConfigKey::FabricSessionToken => self.fabric_session_token.clone(), AzureConfigKey::FabricClusterIdentifier => self.fabric_cluster_identifier.clone(), + AzureConfigKey::Capabilities => self.capabilities.as_ref().map(ToString::to_string), } } @@ -912,7 +919,7 @@ impl MicrosoftAzureBuilder { /// Override the [`Capabilities`] advertised by this store. pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { - self.capabilities = Some(capabilities); + self.capabilities = Some(ConfigValue::Parsed(capabilities)); self } @@ -1066,7 +1073,7 @@ impl MicrosoftAzureBuilder { Ok(MicrosoftAzure { client, - capabilities: self.capabilities, + capabilities: self.capabilities.map(|x| x.get()).transpose()?, }) } } diff --git a/src/gcp/builder.rs b/src/gcp/builder.rs index a36da995..ba91105f 100644 --- a/src/gcp/builder.rs +++ b/src/gcp/builder.rs @@ -123,7 +123,7 @@ pub struct GoogleCloudStorageBuilder { /// The [`HttpConnector`] to use http_connector: Option>, /// Capabilities to advertise for this store instance - capabilities: Option, + capabilities: Option>, } /// Configuration keys for [`GoogleCloudStorageBuilder`] @@ -203,6 +203,9 @@ pub enum GoogleConfigKey { /// Client options Client(ClientConfigKey), + + /// Override the capabilities advertised by this store. + Capabilities, } impl AsRef for GoogleConfigKey { @@ -216,6 +219,7 @@ impl AsRef for GoogleConfigKey { Self::BearerToken => "google_bearer_token", Self::SkipSignature => "google_skip_signature", Self::Client(key) => key.as_ref(), + Self::Capabilities => "google_capabilities", } } } @@ -237,6 +241,7 @@ impl FromStr for GoogleConfigKey { } "google_bearer_token" | "bearer_token" => Ok(Self::BearerToken), "google_skip_signature" | "skip_signature" => Ok(Self::SkipSignature), + "google_capabilities" => Ok(Self::Capabilities), _ => match s.strip_prefix("google_").unwrap_or(s).parse() { Ok(key) => Ok(Self::Client(key)), Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), @@ -345,7 +350,10 @@ impl GoogleCloudStorageBuilder { GoogleConfigKey::SkipSignature => self.skip_signature.parse(value), GoogleConfigKey::Client(key) => { self.client_options = self.client_options.with_config(key, value) - } + }, + GoogleConfigKey::Capabilities => { + self.capabilities = Some(ConfigValue::Deferred(value.into())) + }, }; self } @@ -371,6 +379,7 @@ impl GoogleCloudStorageBuilder { GoogleConfigKey::BearerToken => self.bearer_token.clone(), GoogleConfigKey::SkipSignature => Some(self.skip_signature.to_string()), GoogleConfigKey::Client(key) => self.client_options.get_config_value(key), + GoogleConfigKey::Capabilities => self.capabilities.as_ref().map(|v| v.to_string()), } } @@ -541,7 +550,7 @@ impl GoogleCloudStorageBuilder { /// Override the [`Capabilities`] advertised by this store. pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self { - self.capabilities = Some(capabilities); + self.capabilities = Some(ConfigValue::Parsed(capabilities)); self } @@ -680,7 +689,7 @@ impl GoogleCloudStorageBuilder { let http_client = http.connect(&config.client_options)?; Ok(GoogleCloudStorage { client: Arc::new(GoogleCloudStorageClient::new(config, http_client)?), - capabilities: self.capabilities, + capabilities: self.capabilities.map(|x| x.get()).transpose()?, }) } } From 4094f6b9a9ecb313eae7a847e974fa38879c7843 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Wed, 10 Jun 2026 14:51:38 -0700 Subject: [PATCH 11/14] wip --- src/aws/mod.rs | 1 + src/azure/mod.rs | 1 + src/capabilities.rs | 13 ++++++------- src/gcp/mod.rs | 1 + 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 520d2a38..f9d18296 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -80,6 +80,7 @@ use crate::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore} pub use credential::{AwsAuthorizer, AwsCredential}; /// OrderedListing capability depends on the bucket type, it's not enabled for directory bucket. +/// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html fn get_default_capabilities() -> Capabilities { return Capabilities::new([]); } diff --git a/src/azure/mod.rs b/src/azure/mod.rs index 1b0d3881..cd64565e 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -59,6 +59,7 @@ pub use credential::AzureCredential; const STORE: &str = "MicrosoftAzure"; // OrderedListing capability is not supported by with Azure Storage Hierarchical Namespace is enabled. +// https://learn.microsoft.com/en-us/rest/api/storageservices/list-blobs fn get_default_capabilities() -> Capabilities { Capabilities::new([]) } diff --git a/src/capabilities.rs b/src/capabilities.rs index b23f0985..8970012c 100644 --- a/src/capabilities.rs +++ b/src/capabilities.rs @@ -21,7 +21,7 @@ use crate::Error; use std::collections::HashSet; -const ORDERED_LISTING: &str = "ordered-listing"; +const ORDERED_LISTING: &str = "ordered_listing"; /// An individual capability that an [`ObjectStore`] implementation may support. /// @@ -42,9 +42,7 @@ pub enum Capability { impl std::str::FromStr for Capability { type Err = Error; - /// Parses a capability from its kebab-case string representation. - /// - /// Returns `None` if `s` does not correspond to any known capability. + /// Parses a capability from its snake_case string representation. fn from_str(s: &str) -> Result { match s { ORDERED_LISTING => Ok(Self::OrderedListing), @@ -129,11 +127,12 @@ impl std::str::FromStr for Capabilities { impl std::fmt::Display for Capabilities { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut iter = self.supported.iter(); - if let Some(cap) = iter.next() { + let mut caps: Vec<_> = self.supported.iter().collect(); + caps.sort_by_key(|cap| cap.to_string()); + if let Some(cap) = caps.first() { write!(f, "{}", cap)?; } - for cap in iter { + for cap in caps[1..].iter() { write!(f, ", {}", cap)?; } Ok(()) diff --git a/src/gcp/mod.rs b/src/gcp/mod.rs index 74ddcec9..f7407210 100644 --- a/src/gcp/mod.rs +++ b/src/gcp/mod.rs @@ -67,6 +67,7 @@ mod credential; const STORE: &str = "GCS"; // OrderedListing is supported by all GCP bucket types. +// https://docs.cloud.google.com/storage/docs/listing-objects fn get_default_capabilities() -> Capabilities { Capabilities::new([Capability::OrderedListing]) } From 319d03aba696c238b68728c75f79dc48b0b76d56 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Wed, 10 Jun 2026 14:53:02 -0700 Subject: [PATCH 12/14] fix tests --- src/aws/builder.rs | 4 ++-- src/capabilities.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aws/builder.rs b/src/aws/builder.rs index e85a74ee..3dfc79b8 100644 --- a/src/aws/builder.rs +++ b/src/aws/builder.rs @@ -1658,7 +1658,7 @@ mod tests { "aws_sse_customer_key_base64".parse().unwrap(), "some_customer_key", ) - .with_config(AmazonS3ConfigKey::Capabilities, "ordered-listing"); + .with_config(AmazonS3ConfigKey::Capabilities, "ordered_listing"); assert_eq!( builder @@ -1722,7 +1722,7 @@ mod tests { builder .get_config_value(&"aws_capabilities".parse().unwrap()) .unwrap(), - "ordered-listing" + "ordered_listing" ); } diff --git a/src/capabilities.rs b/src/capabilities.rs index 8970012c..56b9532e 100644 --- a/src/capabilities.rs +++ b/src/capabilities.rs @@ -145,10 +145,10 @@ mod tests { #[test] fn test_capability() { - assert_eq!(format!("{}", Capability::OrderedListing), "ordered-listing"); + assert_eq!(format!("{}", Capability::OrderedListing), "ordered_listing"); assert_eq!( Capability::OrderedListing, - "ordered-listing".parse::().unwrap() + "ordered_listing".parse::().unwrap() ); assert_eq!("invalid".parse::().is_ok(), false); } @@ -163,7 +163,7 @@ mod tests { false ); assert_eq!( - "ordered-listing" + "ordered_listing" .parse::() .unwrap() .has(Capability::OrderedListing), From 46a7a1ad34af15330e281c5961af2bcf0d68c315 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Wed, 10 Jun 2026 15:04:21 -0700 Subject: [PATCH 13/14] update aws, azure and gcp builder tests --- src/aws/builder.rs | 2 +- src/azure/builder.rs | 27 ++++++++++++++++++++++++++- src/gcp/builder.rs | 16 ++++++++++++++++ src/memory.rs | 1 + 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/aws/builder.rs b/src/aws/builder.rs index 3dfc79b8..2a807fb3 100644 --- a/src/aws/builder.rs +++ b/src/aws/builder.rs @@ -1575,7 +1575,7 @@ mod tests { ("aws_session_token", aws_session_token.clone()), ("aws_unsigned_payload", "true".to_string()), ("aws_checksum_algorithm", "sha256".to_string()), - ("aws_capabilities", "ordered-listing".to_string()), + ("aws_capabilities", "ordered_listing".to_string()), ]); let builder = options diff --git a/src/azure/builder.rs b/src/azure/builder.rs index e8d5b17f..da63d449 100644 --- a/src/azure/builder.rs +++ b/src/azure/builder.rs @@ -23,7 +23,10 @@ use crate::azure::credential::{ use crate::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzure, STORE}; use crate::client::{HttpConnector, TokenCredentialProvider, http_connector}; use crate::config::ConfigValue; -use crate::{Capabilities, ClientConfigKey, ClientOptions, ObjectStoreExt, Result, RetryConfig, StaticCredentialProvider}; +use crate::{ + Capabilities, ClientConfigKey, ClientOptions, ObjectStoreExt, Result, RetryConfig, + StaticCredentialProvider, +}; use percent_encoding::percent_decode_str; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -1117,6 +1120,7 @@ pub fn split_sas(sas: &str) -> Result> { #[cfg(test)] mod tests { use super::*; + use crate::Capability; use std::collections::HashMap; #[test] @@ -1264,6 +1268,7 @@ mod tests { ("azure_client_id", azure_client_id), ("azure_storage_account_name", azure_storage_account_name), ("azure_storage_token", azure_storage_token), + ("azure_capabilities", "ordered_listing"), ]); let builder = options @@ -1274,6 +1279,26 @@ mod tests { assert_eq!(builder.client_id.unwrap(), azure_client_id); assert_eq!(builder.account_name.unwrap(), azure_storage_account_name); assert_eq!(builder.bearer_token.unwrap(), azure_storage_token); + assert!( + builder + .capabilities + .unwrap() + .get() + .unwrap() + .has(Capability::OrderedListing) + ); + } + + #[test] + fn azure_test_config_get_value() { + let builder = MicrosoftAzureBuilder::new() + .with_config(AzureConfigKey::Capabilities, "ordered_listing"); + assert_eq!( + builder + .get_config_value(&"azure_capabilities".parse().unwrap()) + .unwrap(), + "ordered_listing" + ); } #[test] diff --git a/src/gcp/builder.rs b/src/gcp/builder.rs index ba91105f..154bd86c 100644 --- a/src/gcp/builder.rs +++ b/src/gcp/builder.rs @@ -697,6 +697,7 @@ impl GoogleCloudStorageBuilder { #[cfg(test)] mod tests { use super::*; + use crate::Capability; use std::collections::HashMap; use std::io::Write; use tempfile::NamedTempFile; @@ -723,6 +724,7 @@ mod tests { let options = HashMap::from([ ("google_service_account", google_service_account.clone()), ("google_bucket_name", google_bucket_name.clone()), + ("google_capabilities", "ordered_listing".to_string()), ]); let builder = options @@ -736,6 +738,14 @@ mod tests { google_service_account.as_str() ); assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str()); + assert!( + builder + .capabilities + .unwrap() + .get() + .unwrap() + .has(Capability::OrderedListing) + ); } #[tokio::test] @@ -894,6 +904,12 @@ mod tests { .unwrap(), google_bearer_token ); + assert_eq!( + builder + .get_config_value(&"google_capabilities".parse().unwrap()) + .unwrap(), + "ordered_listing" + ); } #[test] diff --git a/src/memory.rs b/src/memory.rs index c8fed468..0e91db84 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -551,6 +551,7 @@ mod tests { #[tokio::test] async fn in_memory_test() { let integration = InMemory::new(); + assert!(integration.capabilities().has(Capability::OrderedListing)); put_get_delete_list(&integration).await; list_with_offset_exclusivity(&integration).await; From 6c81f6a4c036051f5681541b20096e79eed753f3 Mon Sep 17 00:00:00 2001 From: Victor Ordaz Date: Wed, 10 Jun 2026 17:03:32 -0700 Subject: [PATCH 14/14] wip --- src/aws/mod.rs | 4 ++-- src/azure/mod.rs | 2 +- src/capabilities.rs | 12 +++--------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/aws/mod.rs b/src/aws/mod.rs index f9d18296..293bfcb6 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -79,8 +79,8 @@ use crate::client::parts::Parts; use crate::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; pub use credential::{AwsAuthorizer, AwsCredential}; -/// OrderedListing capability depends on the bucket type, it's not enabled for directory bucket. -/// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html +// OrderedListing capability depends on the bucket type, it's not enabled for directory bucket. +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html fn get_default_capabilities() -> Capabilities { return Capabilities::new([]); } diff --git a/src/azure/mod.rs b/src/azure/mod.rs index cd64565e..4264004f 100644 --- a/src/azure/mod.rs +++ b/src/azure/mod.rs @@ -24,7 +24,7 @@ //! Unused blocks will automatically be dropped after 7 days. //! use crate::{ - Capabilities, Capability, CopyMode, CopyOptions, GetOptions, GetResult, ListResult, + Capabilities, CopyMode, CopyOptions, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart, multipart::{MultipartStore, PartId}, diff --git a/src/capabilities.rs b/src/capabilities.rs index 56b9532e..77762541 100644 --- a/src/capabilities.rs +++ b/src/capabilities.rs @@ -127,15 +127,9 @@ impl std::str::FromStr for Capabilities { impl std::fmt::Display for Capabilities { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut caps: Vec<_> = self.supported.iter().collect(); - caps.sort_by_key(|cap| cap.to_string()); - if let Some(cap) = caps.first() { - write!(f, "{}", cap)?; - } - for cap in caps[1..].iter() { - write!(f, ", {}", cap)?; - } - Ok(()) + let mut caps: Vec<_> = self.supported.iter().map(ToString::to_string).collect(); + caps.sort(); + write!(f, "{}", caps.join(", ")) } }