refactor(buffered): drop ObjectMeta from ReadableFile

Author: fvaleye

obstore/python/obstore/_buffered.pyi

@@ -3,7 +3,6 @@ from contextlib import AbstractAsyncContextManager, AbstractContextManager
 
 from ._attributes import Attributes
 from ._bytes import Bytes
-from ._list import ObjectMeta
 from ._store import ObjectStore
 
 if sys.version_info >= (3, 11):
@@ -33,9 +32,7 @@ def open_reader(
         buffer_size: The minimum number of bytes to read in a single request. Up to `buffer_size` bytes will be buffered in memory.
         size: Optional byte size of the object. When provided, skips the HEAD request used to fetch the file size. Useful for callers that already know the size from external metadata.
 
-            The caller is responsible for accuracy: a value larger than the actual file surfaces as a read-time range error, a value smaller causes silent truncation.
-
-            When `size` is provided, the resulting reader's `meta` attribute omits `last_modified` (since it was not fetched). Callers that need that field should call `open_reader` without `size`. Defaults to `None`.
+            The caller is responsible for accuracy: a value larger than the actual file surfaces as a read-time range error, a value smaller causes silent truncation. Defaults to `None`.
 
     Returns:
         ReadableFile
@@ -94,10 +91,6 @@ class ReadableFile:
         This is currently a no-op.
         """
 
-    @property
-    def meta(self) -> ObjectMeta:
-        """Access the metadata of the underlying file."""
-
     def read(self, size: int | None = None, /) -> Bytes:
         """Read up to `size` bytes from the object and return them.
 
@@ -176,10 +169,6 @@ class AsyncReadableFile:
         This is currently a no-op.
         """
 
-    @property
-    def meta(self) -> ObjectMeta:
-        """Access the metadata of the underlying file."""
-
     async def read(self, size: int | None = None, /) -> Bytes:
         """Read up to `size` bytes from the object and return them.
 

obstore/src/buffered.rs

@@ -2,13 +2,11 @@ use std::io::SeekFrom;
 use std::sync::Arc;
 
 use bytes::Bytes;
-use chrono::{DateTime, Utc};
-use indexmap::IndexMap;
 use object_store::buffered::{BufReader, BufWriter};
 use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt};
 use pyo3::exceptions::{PyIOError, PyStopAsyncIteration, PyStopIteration};
 use pyo3::prelude::*;
-use pyo3::types::{PyDict, PyString};
+use pyo3::types::PyString;
 use pyo3::{intern, IntoPyObjectExt};
 use pyo3_async_runtimes::tokio::{future_into_py, get_runtime};
 use pyo3_bytes::PyBytes;
@@ -30,10 +28,9 @@ pub(crate) fn open_reader(
 ) -> PyObjectStoreResult<PyReadableFile> {
     let store = store.into_inner();
     let runtime = get_runtime();
-    let was_hinted = size.is_some();
-    let (reader, meta) =
+    let (reader, resolved_size) =
         py.detach(|| runtime.block_on(create_reader(store, path, buffer_size, size)))?;
-    Ok(PyReadableFile::new(reader, meta, was_hinted, false))
+    Ok(PyReadableFile::new(reader, resolved_size, false))
 }
 
 #[pyfunction]
@@ -46,10 +43,9 @@ pub(crate) fn open_reader_async(
     size: Option<u64>,
 ) -> PyResult<Bound<PyAny>> {
     let store = store.into_inner();
-    let was_hinted = size.is_some();
     future_into_py(py, async move {
-        let (reader, meta) = create_reader(store, path, buffer_size, size).await?;
-        Ok(PyReadableFile::new(reader, meta, was_hinted, true))
+        let (reader, resolved_size) = create_reader(store, path, buffer_size, size).await?;
+        Ok(PyReadableFile::new(reader, resolved_size, true))
     })
 }
 
@@ -58,12 +54,11 @@ async fn create_reader(
     path: PyPath,
     capacity: usize,
     size: Option<u64>,
-) -> PyObjectStoreResult<(BufReader, ObjectMeta)> {
+) -> PyObjectStoreResult<(BufReader, u64)> {
     let meta = match size {
         Some(size) => ObjectMeta {
             location: path.as_ref().clone(),
-            last_modified: DateTime::<Utc>::from_timestamp(0, 0)
-                .expect("unix epoch is a valid DateTime"),
+            last_modified: Default::default(),
             size,
             e_tag: None,
             version: None,
@@ -73,23 +68,22 @@ async fn create_reader(
             .await
             .map_err(PyObjectStoreError::ObjectStoreError)?,
     };
-    Ok((BufReader::with_capacity(store, &meta, capacity), meta))
+    let size = meta.size;
+    Ok((BufReader::with_capacity(store, &meta, capacity), size))
 }
 
 #[pyclass(name = "ReadableFile", frozen)]
 pub(crate) struct PyReadableFile {
     reader: Arc<Mutex<BufReader>>,
-    meta: ObjectMeta,
-    was_hinted: bool,
+    size: u64,
     r#async: bool,
 }
 
 impl PyReadableFile {
-    fn new(reader: BufReader, meta: ObjectMeta, was_hinted: bool, r#async: bool) -> Self {
+    fn new(reader: BufReader, size: u64, r#async: bool) -> Self {
         Self {
             reader: Arc::new(Mutex::new(reader)),
-            meta,
-            was_hinted,
+            size,
             r#async,
         }
     }
@@ -110,22 +104,6 @@ impl PyReadableFile {
     // `Option<Arc<Mutex<BufReader>>>`.
     fn close(&self) {}
 
-    #[getter]
-    fn meta<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
-        let mut dict = IndexMap::with_capacity(5);
-        dict.insert("path", self.meta.location.as_ref().into_bound_py_any(py)?);
-        if !self.was_hinted {
-            dict.insert(
-                "last_modified",
-                self.meta.last_modified.into_bound_py_any(py)?,
-            );
-        }
-        dict.insert("size", self.meta.size.into_bound_py_any(py)?);
-        dict.insert("e_tag", self.meta.e_tag.clone().into_bound_py_any(py)?);
-        dict.insert("version", self.meta.version.clone().into_bound_py_any(py)?);
-        dict.into_pyobject(py)
-    }
-
     #[pyo3(signature = (size = None, /))]
     fn read<'py>(&'py self, py: Python<'py>, size: Option<usize>) -> PyResult<Bound<'py, PyAny>> {
         let reader = self.reader.clone();
@@ -203,7 +181,7 @@ impl PyReadableFile {
 
     #[getter]
     fn size(&self) -> u64 {
-        self.meta.size
+        self.size
     }
 
     fn tell<'py>(&'py self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {

tests/test_buffered.py

@@ -171,18 +171,11 @@ def test_open_reader_size_hint_zero_byte_file():
     assert memoryview(b"") == memoryview(file.read())
 
 
-def test_open_reader_meta_last_modified_depends_on_size_hint():
+def test_open_reader_no_longer_exposes_meta():
     store = MemoryStore()
     data = b"x" * 1000
     path = "sized.bin"
     obs.put(store, path, data)
 
-    hinted = obs.open_reader(store, path, size=len(data))
-    unhinted = obs.open_reader(store, path)
-
-    assert "last_modified" not in hinted.meta
-    assert "last_modified" in unhinted.meta
-    assert hinted.meta["size"] == len(data)
-    assert unhinted.meta["size"] == len(data)
-    assert hinted.meta["e_tag"] is None
-    assert hinted.meta["path"] == path
+    file = obs.open_reader(store, path)
+    assert not hasattr(file, "meta")

tests/test_fsspec.py

@@ -167,7 +167,6 @@ def test_buffered_file_forwards_size_to_open_reader():
 
         assert file.size == 500
         assert file._reader.size == 500
-        assert "last_modified" not in file._reader.meta
 
         data = file.read()
         assert len(data) == 500