Skip to content

Commit 2853792

Browse files
hadimmartindurant
andauthored
Fix _cat_file / _cat_ranges not discoverable on WholeFileCacheFileSystem class (#2010)
--------- Co-authored-by: Martin Durant <martin.durant@alumni.utoronto.ca>
1 parent cb36bec commit 2853792

2 files changed

Lines changed: 62 additions & 43 deletions

File tree

fsspec/implementations/cached.py

Lines changed: 46 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -474,9 +474,12 @@ def __getattribute__(self, item):
474474
}:
475475
# all the methods defined in this class. Note `open` here, since
476476
# it calls `_open`, but is actually in superclass
477-
return lambda *args, **kw: getattr(type(self), item).__get__(self)(
478-
*args, **kw
479-
)
477+
if hasattr(type(self), item):
478+
return lambda *args, **kw: getattr(type(self), item).__get__(self)(
479+
*args, **kw
480+
)
481+
# method is in the whitelist but not defined on this subclass;
482+
# fall through to delegate to the wrapped filesystem below
480483
if item in ["__reduce_ex__"]:
481484
raise AttributeError
482485
if item in ["transaction"]:
@@ -744,6 +747,46 @@ def _open(self, path, mode="rb", **kwargs):
744747
}
745748
return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs)
746749

750+
async def _cat_file(self, path, start=None, end=None, **kwargs):
751+
logger.debug("async cat_file %s", path)
752+
path = self._strip_protocol(path)
753+
sha = self._mapper(path)
754+
fn = self._check_file(path)
755+
756+
if not fn:
757+
fn = os.path.join(self.storage[-1], sha)
758+
await self.fs._get_file(path, fn, **kwargs)
759+
760+
with open(fn, "rb") as f: # noqa ASYNC230
761+
if start:
762+
f.seek(start)
763+
size = -1 if end is None else end - f.tell()
764+
return f.read(size)
765+
766+
async def _cat_ranges(
767+
self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
768+
):
769+
logger.debug("async cat ranges %s", paths)
770+
lpaths = []
771+
rset = set()
772+
download = []
773+
rpaths = []
774+
for p in paths:
775+
fn = self._check_file(p)
776+
if fn is None and p not in rset:
777+
sha = self._mapper(p)
778+
fn = os.path.join(self.storage[-1], sha)
779+
download.append(fn)
780+
rset.add(p)
781+
rpaths.append(p)
782+
lpaths.append(fn)
783+
if download:
784+
await self.fs._get(rpaths, download, on_error=on_error)
785+
786+
return LocalFileSystem().cat_ranges(
787+
lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
788+
)
789+
747790

748791
class SimpleCacheFileSystem(WholeFileCacheFileSystem):
749792
"""Caches whole remote files on first access
@@ -848,46 +891,6 @@ def pipe(self, path, value=None, **kwargs):
848891
else:
849892
raise ValueError("path must be str or dict")
850893

851-
async def _cat_file(self, path, start=None, end=None, **kwargs):
852-
logger.debug("async cat_file %s", path)
853-
path = self._strip_protocol(path)
854-
sha = self._mapper(path)
855-
fn = self._check_file(path)
856-
857-
if not fn:
858-
fn = os.path.join(self.storage[-1], sha)
859-
await self.fs._get_file(path, fn, **kwargs)
860-
861-
with open(fn, "rb") as f: # noqa ASYNC230
862-
if start:
863-
f.seek(start)
864-
size = -1 if end is None else end - f.tell()
865-
return f.read(size)
866-
867-
async def _cat_ranges(
868-
self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
869-
):
870-
logger.debug("async cat ranges %s", paths)
871-
lpaths = []
872-
rset = set()
873-
download = []
874-
rpaths = []
875-
for p in paths:
876-
fn = self._check_file(p)
877-
if fn is None and p not in rset:
878-
sha = self._mapper(p)
879-
fn = os.path.join(self.storage[-1], sha)
880-
download.append(fn)
881-
rset.add(p)
882-
rpaths.append(p)
883-
lpaths.append(fn)
884-
if download:
885-
await self.fs._get(rpaths, download, on_error=on_error)
886-
887-
return LocalFileSystem().cat_ranges(
888-
lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
889-
)
890-
891894
def cat_ranges(
892895
self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
893896
):

fsspec/implementations/tests/test_cached.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1440,3 +1440,19 @@ def test_simplecache_instance_cache(instance_caches):
14401440
"file": 1,
14411441
"http": 1,
14421442
}
1443+
1444+
1445+
@pytest.mark.parametrize("protocol", ["filecache", "simplecache"])
1446+
def test_class_has_cat_file_and_cat_ranges(tmp_path, protocol):
1447+
"""Ensure _cat_file and _cat_ranges are available on the class, not just
1448+
instances, so that external code inspecting ``type(fs)`` (e.g.
1449+
universal_pathlib, zarr) can discover these capabilities.
1450+
1451+
Regression test for https://github.com/fsspec/filesystem_spec/issues/2009
1452+
"""
1453+
fs = fsspec.filesystem(
1454+
protocol, target_protocol="memory", cache_storage=str(tmp_path)
1455+
)
1456+
for attr in ("_cat_file", "_cat_ranges"):
1457+
assert hasattr(fs, attr), f"instance missing {attr}"
1458+
assert hasattr(type(fs), attr), f"class missing {attr}"

0 commit comments

Comments
 (0)