feat: upgrade object store 0.13.x (#600)

alukach · web-flow · commit a2ca349acf48 · 2026-01-12T22:20:51.000Z
* bump dependencies

* Update error shape

* Embrace ObjectStoreExt
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -26,7 +26,7 @@ chrono = "0.4.38"
 futures = "0.3.31"
 http = "1.2"
 indexmap = "2"
-object_store = "0.12.4"
+object_store = { version = "0.13.0", features = ["aws", "azure", "gcp", "http"] }
 pyo3 = { version = "0.27.1", features = ["macros", "indexmap"] }
 pyo3-async-runtimes = { version = "0.27", features = ["tokio-runtime"] }
 pyo3-file = { git = "https://github.com/kylebarron/pyo3-file", rev = "aacc18816591f9987247bac8b7011b452b4eeb3e" }
diff --git a/obstore/src/buffered.rs b/obstore/src/buffered.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use bytes::Bytes;
 use object_store::buffered::{BufReader, BufWriter};
-use object_store::{ObjectMeta, ObjectStore};
+use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt};
 use pyo3::exceptions::{PyIOError, PyStopAsyncIteration, PyStopIteration};
 use pyo3::prelude::*;
 use pyo3::types::PyString;
diff --git a/obstore/src/copy.rs b/obstore/src/copy.rs
@@ -1,4 +1,6 @@
-use object_store::ObjectStore;
+use std::{future::Future, pin::Pin};
+
+use object_store::ObjectStoreExt;
 use pyo3::prelude::*;
 use pyo3_async_runtimes::tokio::get_runtime;
 use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult};
@@ -18,10 +20,10 @@ pub(crate) fn copy(
     let from_ = from_.into();
     let to = to.into();
     py.detach(|| {
-        let fut = if overwrite {
-            store.as_ref().copy(&from_, &to)
+        let fut: Pin<Box<dyn Future<Output = _> + Send>> = if overwrite {
+            Box::pin(store.as_ref().copy(&from_, &to))
         } else {
-            store.as_ref().copy_if_not_exists(&from_, &to)
+            Box::pin(store.as_ref().copy_if_not_exists(&from_, &to))
         };
         runtime.block_on(fut)?;
         Ok::<_, PyObjectStoreError>(())
@@ -40,10 +42,10 @@ pub(crate) fn copy_async(
     let from_ = from_.into();
     let to = to.into();
     pyo3_async_runtimes::tokio::future_into_py(py, async move {
-        let fut = if overwrite {
-            store.as_ref().copy(&from_, &to)
+        let fut: Pin<Box<dyn Future<Output = _> + Send>> = if overwrite {
+            Box::pin(store.as_ref().copy(&from_, &to))
         } else {
-            store.as_ref().copy_if_not_exists(&from_, &to)
+            Box::pin(store.as_ref().copy_if_not_exists(&from_, &to))
         };
         fut.await.map_err(PyObjectStoreError::ObjectStoreError)?;
         Ok(PyNone)
diff --git a/obstore/src/delete.rs b/obstore/src/delete.rs
@@ -1,4 +1,5 @@
 use futures::{StreamExt, TryStreamExt};
+use object_store::ObjectStoreExt;
 use pyo3::prelude::*;
 use pyo3_async_runtimes::tokio::get_runtime;
 use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult};
diff --git a/obstore/src/get.rs b/obstore/src/get.rs
@@ -6,7 +6,9 @@ use bytes::Bytes;
 use chrono::{DateTime, Utc};
 use futures::stream::{BoxStream, Fuse};
 use futures::StreamExt;
-use object_store::{Attributes, GetOptions, GetRange, GetResult, ObjectMeta, ObjectStore};
+use object_store::{
+    Attributes, GetOptions, GetRange, GetResult, ObjectMeta, ObjectStore, ObjectStoreExt,
+};
 use pyo3::exceptions::{PyStopAsyncIteration, PyStopIteration, PyValueError};
 use pyo3::prelude::*;
 use pyo3_async_runtimes::tokio::get_runtime;
@@ -347,7 +349,7 @@ pub(crate) fn get(
         let fut = if let Some(options) = options {
             store.as_ref().get_opts(path, options.into())
         } else {
-            store.as_ref().get(path)
+            Box::pin(store.as_ref().get(path))
         };
         let out = runtime.block_on(fut)?;
         Ok::<_, PyObjectStoreError>(PyGetResult::new(out))
@@ -367,7 +369,7 @@ pub(crate) fn get_async(
         let fut = if let Some(options) = options {
             store.as_ref().get_opts(path, options.into())
         } else {
-            store.as_ref().get(path)
+            Box::pin(store.as_ref().get(path))
         };
         let out = fut.await.map_err(PyObjectStoreError::ObjectStoreError)?;
         Ok(PyGetResult::new(out))
diff --git a/obstore/src/head.rs b/obstore/src/head.rs
@@ -1,3 +1,4 @@
+use object_store::ObjectStoreExt;
 use pyo3::prelude::*;
 use pyo3_async_runtimes::tokio::get_runtime;
 use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult, PyPath};
diff --git a/obstore/src/rename.rs b/obstore/src/rename.rs
@@ -1,4 +1,6 @@
-use object_store::ObjectStore;
+use std::{future::Future, pin::Pin};
+
+use object_store::ObjectStoreExt;
 use pyo3::prelude::*;
 use pyo3_async_runtimes::tokio::get_runtime;
 use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult};
@@ -18,10 +20,10 @@ pub(crate) fn rename(
     let from_ = from_.into();
     let to = to.into();
     py.detach(|| {
-        let fut = if overwrite {
-            store.as_ref().rename(&from_, &to)
+        let fut: Pin<Box<dyn Future<Output = _> + Send>> = if overwrite {
+            Box::pin(store.as_ref().rename(&from_, &to))
         } else {
-            store.as_ref().rename_if_not_exists(&from_, &to)
+            Box::pin(store.as_ref().rename_if_not_exists(&from_, &to))
         };
         runtime.block_on(fut)?;
         Ok::<_, PyObjectStoreError>(())
@@ -40,10 +42,10 @@ pub(crate) fn rename_async(
     let from_ = from_.into();
     let to = to.into();
     pyo3_async_runtimes::tokio::future_into_py(py, async move {
-        let fut = if overwrite {
-            store.as_ref().rename(&from_, &to)
+        let fut: Pin<Box<dyn Future<Output = _> + Send>> = if overwrite {
+            Box::pin(store.as_ref().rename(&from_, &to))
         } else {
-            store.as_ref().rename_if_not_exists(&from_, &to)
+            Box::pin(store.as_ref().rename_if_not_exists(&from_, &to))
         };
         fut.await.map_err(PyObjectStoreError::ObjectStoreError)?;
         Ok(PyNone)
diff --git a/pyo3-object_store/Cargo.toml b/pyo3-object_store/Cargo.toml
@@ -28,7 +28,7 @@ humantime = "2.1"
 http = "1"
 # This is already an object_store dependency
 itertools = "0.14.0"
-object_store = { version = "0.12.4", features = [
+object_store = { version = "0.13.0", features = [
     "aws",
     "azure",
     "gcp",
diff --git a/pyo3-object_store/src/error.rs b/pyo3-object_store/src/error.rs
@@ -131,9 +131,10 @@ impl From<PyObjectStoreError> for PyErr {
                 object_store::Error::NotModified { path: _, source: _ } => {
                     NotModifiedError::new_err(print_with_debug(err))
                 }
-                object_store::Error::NotImplemented => {
-                    PyNotImplementedError::new_err(print_with_debug(err))
-                }
+                object_store::Error::NotImplemented {
+                    operation: _,
+                    implementer: _,
+                } => PyNotImplementedError::new_err(print_with_debug(err)),
                 object_store::Error::PermissionDenied { path: _, source: _ } => {
                     PermissionDeniedError::new_err(print_with_debug(err))
                 }
diff --git a/pyo3-object_store/src/prefix.rs b/pyo3-object_store/src/prefix.rs
@@ -9,7 +9,7 @@ use std::borrow::Cow;
 use std::ops::Range;
 use std::sync::OnceLock;
 
-use object_store::path::Path;
+use object_store::{path::Path, CopyOptions};
 // Remove when updating to object_store 0.13
 #[allow(deprecated)]
 use object_store::{
@@ -112,11 +112,6 @@ fn strip_meta(prefix: Option<&Path>, meta: ObjectMeta) -> ObjectMeta {
 }
 #[async_trait::async_trait]
 impl<T: ObjectStore> ObjectStore for MaybePrefixedStore<T> {
-    async fn put(&self, location: &Path, payload: PutPayload) -> Result<PutResult> {
-        let full_path = self.full_path(location);
-        self.inner.put(&full_path, payload).await
-    }
-
     async fn put_opts(
         &self,
         location: &Path,
@@ -127,11 +122,6 @@ impl<T: ObjectStore> ObjectStore for MaybePrefixedStore<T> {
         self.inner.put_opts(&full_path, payload, opts).await
     }
 
-    async fn put_multipart(&self, location: &Path) -> Result<Box<dyn MultipartUpload>> {
-        let full_path = self.full_path(location);
-        self.inner.put_multipart(&full_path).await
-    }
-
     // Remove when updating to object_store 0.13
     #[allow(deprecated)]
     async fn put_multipart_opts(
@@ -143,16 +133,6 @@ impl<T: ObjectStore> ObjectStore for MaybePrefixedStore<T> {
         self.inner.put_multipart_opts(&full_path, opts).await
     }
 
-    async fn get(&self, location: &Path) -> Result<GetResult> {
-        let full_path = self.full_path(location);
-        self.inner.get(&full_path).await
-    }
-
-    async fn get_range(&self, location: &Path, range: Range<u64>) -> Result<Bytes> {
-        let full_path = self.full_path(location);
-        self.inner.get_range(&full_path, range).await
-    }
-
     async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
         let full_path = self.full_path(location);
         self.inner.get_opts(&full_path, options).await
@@ -163,17 +143,6 @@ impl<T: ObjectStore> ObjectStore for MaybePrefixedStore<T> {
         self.inner.get_ranges(&full_path, ranges).await
     }
 
-    async fn head(&self, location: &Path) -> Result<ObjectMeta> {
-        let full_path = self.full_path(location);
-        let meta = self.inner.head(&full_path).await?;
-        Ok(self.strip_meta(meta))
-    }
-
-    async fn delete(&self, location: &Path) -> Result<()> {
-        let full_path = self.full_path(location);
-        self.inner.delete(&full_path).await
-    }
-
     fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
         let prefix = self.full_path(prefix.unwrap_or(DEFAULT_PATH.get_or_init(Path::default)));
         let s = self.inner.list(Some(&prefix));
@@ -214,27 +183,16 @@ impl<T: ObjectStore> ObjectStore for MaybePrefixedStore<T> {
             })
     }
 
-    async fn copy(&self, from: &Path, to: &Path) -> Result<()> {
-        let full_from = self.full_path(from);
-        let full_to = self.full_path(to);
-        self.inner.copy(&full_from, &full_to).await
+    async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
+        let from_full = self.full_path(from);
+        let to_full = self.full_path(to);
+        self.inner.copy_opts(&from_full, &to_full, options).await
     }
 
-    async fn rename(&self, from: &Path, to: &Path) -> Result<()> {
-        let full_from = self.full_path(from);
-        let full_to = self.full_path(to);
-        self.inner.rename(&full_from, &full_to).await
-    }
-
-    async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> {
-        let full_from = self.full_path(from);
-        let full_to = self.full_path(to);
-        self.inner.copy_if_not_exists(&full_from, &full_to).await
-    }
-
-    async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> {
-        let full_from = self.full_path(from);
-        let full_to = self.full_path(to);
-        self.inner.rename_if_not_exists(&full_from, &full_to).await
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, Result<Path>>,
+    ) -> BoxStream<'static, Result<Path>> {
+        self.inner.delete_stream(locations)
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`use futures::{StreamExt, TryStreamExt};`
	`2`	`+use object_store::ObjectStoreExt;`
`2`	`3`	`use pyo3::prelude::*;`
`3`	`4`	`use pyo3_async_runtimes::tokio::get_runtime;`
`4`	`5`	`use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult};`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+use object_store::ObjectStoreExt;`
`1`	`2`	`use pyo3::prelude::*;`
`2`	`3`	`use pyo3_async_runtimes::tokio::get_runtime;`
`3`	`4`	`use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult, PyPath};`
Original file line number	Diff line number	Diff line change
`@@ -131,9 +131,10 @@ impl From<PyObjectStoreError> for PyErr {`
`131`	`131`	`object_store::Error::NotModified { path: _, source: _ } => {`
`132`	`132`	`NotModifiedError::new_err(print_with_debug(err))`
`133`	`133`	`}`
`134`		`- object_store::Error::NotImplemented => {`
`135`		`- PyNotImplementedError::new_err(print_with_debug(err))`
`136`		`- }`
	`134`	`+ object_store::Error::NotImplemented {`
	`135`	`+ operation: _,`
	`136`	`+ implementer: _,`
	`137`	`+ } => PyNotImplementedError::new_err(print_with_debug(err)),`
`137`	`138`	`object_store::Error::PermissionDenied { path: _, source: _ } => {`
`138`	`139`	`PermissionDeniedError::new_err(print_with_debug(err))`
`139`	`140`	`}`