feat(vortex-python): Add OpenDAL-backed CosStore to the Python object-store API

Author: XuQianJin-Stars

Cargo.lock

@@ -11,6 +11,7 @@ members = [
     "vortex-flatbuffers",
     "vortex-metrics",
     "vortex-io",
+    "vortex-object-store-opendal",
     "vortex-proto",
     "vortex-array",
     "vortex-arrow",
@@ -429,3 +430,12 @@ incremental = false
 # This improved build times significantly for default common cases that we use locally
 [profile.dev.package.vortex-fastlanes]
 debug = false
+
+# Vendored fork of `pyo3-object_store` that adds `From<Arc<dyn ObjectStore>> for
+# PyObjectStore`, allowing any pre-built `object_store::ObjectStore` (e.g. the OpenDAL-backed
+# COS/OSS store) to be passed to Vortex's `read_url(store=...)` / `write(store=...)` APIs.
+#
+# Once https://github.com/developmentseed/obstore (pyo3-object_store) ships this constructor
+# upstream, this patch can be removed.
+[patch.crates-io]
+pyo3-object_store = { path = "vendor/pyo3-object_store" }

Cargo.toml

@@ -13,6 +13,7 @@ Vortex arrays support reading and writing to many object storage systems:
    store/http
    store/local
    store/memory
+   store/opendal
    store/config
 
 .. autofunction:: vortex.store.from_url

docs/api/python/store.rst

@@ -0,0 +1,62 @@
+========
+OpenDAL (COS)
+========
+
+Vortex can read from and write to Tencent Cloud COS through
+`OpenDAL <https://opendal.apache.org/>`_, which provides native service support.
+
+This store is available only when Vortex is built with the ``opendal`` feature
+(e.g. ``maturin develop --features opendal`` or ``cargo build -p vortex-jni --features opendal``).
+
+.. autoclass:: vortex.store.CosStore
+   :members:
+
+Reading from COS
+================
+
+Pass a ``cos://`` URL directly. Credentials and the endpoint are picked up from the environment
+variables OpenDAL's COS builder reads (``TENCENTCLOUD_SECRET_ID``, ``TENCENTCLOUD_SECRET_KEY`` and
+``COS_ENDPOINT``):
+
+.. code-block:: python
+
+   import vortex as vx
+
+   a = vx.io.read_url("cos://my-bucket/path/to/dataset.vortex")
+
+Or configure explicitly with :class:`~vortex.store.CosStore` before reading:
+
+.. code-block:: python
+
+   from vortex.store import CosStore
+
+   CosStore(
+       bucket="my-bucket",
+       endpoint="https://cos.ap-guangzhou.myqcloud.com",
+       secret_id="AKID...",
+       secret_key="...",
+   ).apply()
+
+   a = vx.io.read_url("cos://my-bucket/path/to/dataset.vortex")
+
+Passing a store object directly
+===============================
+
+``CosStore`` is a concrete store object. You can build one once and pass it
+directly to :func:`vortex.io.read_url` / :func:`vortex.io.write` via the ``store=`` argument,
+exactly like the built-in S3/Azure/GCS stores:
+
+.. code-block:: python
+
+   from vortex.io import read_url
+   from vortex.store import CosStore
+
+   store = CosStore(
+       bucket="my-bucket",
+       endpoint="https://cos.ap-guangzhou.myqcloud.com",
+       secret_id="AKID...",
+       secret_key="...",
+   )
+
+   a = read_url("cos://my-bucket/path/to/dataset.vortex", store=store)
+

docs/api/python/store/opendal.rst

@@ -0,0 +1,47 @@
+[package]
+name = "pyo3-object_store"
+version = "0.11.0"
+authors = ["Kyle Barron <kyle@developmentseed.org>"]
+edition = "2021"
+description = "object_store integration for pyo3."
+readme = "README.md"
+repository = "https://github.com/developmentseed/obstore"
+license = "MIT OR Apache-2.0"
+keywords = []
+categories = []
+rust-version = "1.75"
+# Include the Python type hints as part of the cargo distribution
+include = ["src", "type-hints", "README.md", "LICENSE"]
+
+[features]
+default = ["external-store-warning"]
+external-store-warning = []
+
+[dependencies]
+async-trait = "0.1.85"
+bytes = "1"
+chrono = "0.4"
+futures = "0.3"
+# This is already an object_store dependency
+humantime = "2.1"
+# This is already an object_store dependency
+http = "1"
+# This is already an object_store dependency
+itertools = "0.14.0"
+object_store = { version = "0.13.0", features = [
+    "aws",
+    "azure",
+    "gcp",
+    "http",
+] }
+# This is already an object_store dependency
+percent-encoding = "2.1"
+pyo3 = { version = "0.29", features = ["chrono", "indexmap"] }
+pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] }
+serde = "1"
+thiserror = "1"
+tokio = { version = "1.40", features = ["rt-multi-thread"] }
+url = "2"
+
+[lib]
+crate-type = ["rlib"]

vendor/pyo3-object_store/Cargo.toml

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Development Seed
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.