Skip to content

Commit 53b5002

Browse files
kushalbakshiclaude
andcommitted
feat: wire StorageAdapter delegation into StorageBackend
The three else-branches in StorageBackend (_create_filesystem, _full_path, get_url) now query the adapter registry before falling back to the built-in behaviour. Registered plugin adapters are called for filesystem construction, path composition, and URL generation; unknown protocols still raise DataJointError. Includes new test class TestStorageBackendPluginDelegation. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent b8de1d8 commit 53b5002

File tree

2 files changed

+92
-3
lines changed

2 files changed

+92
-3
lines changed

src/datajoint/storage.py

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,14 @@ def _create_filesystem(self) -> fsspec.AbstractFileSystem:
368368
)
369369

370370
else:
371-
raise errors.DataJointError(f"Unsupported storage protocol: {self.protocol}")
371+
from .storage_adapter import get_storage_adapter
372+
373+
adapter = get_storage_adapter(self.protocol)
374+
if adapter is None:
375+
raise errors.DataJointError(
376+
f"Unsupported storage protocol: {self.protocol}"
377+
)
378+
return adapter.create_filesystem(self.spec)
372379

373380
def _full_path(self, path: str | PurePosixPath) -> str:
374381
"""
@@ -398,7 +405,12 @@ def _full_path(self, path: str | PurePosixPath) -> str:
398405
return f"{bucket}/{location}/{path}"
399406
return f"{bucket}/{path}"
400407
else:
401-
# Local filesystem - prepend location if specified
408+
from .storage_adapter import get_storage_adapter
409+
410+
adapter = get_storage_adapter(self.protocol)
411+
if adapter is not None:
412+
return adapter.full_path(self.spec, path)
413+
# File-protocol fallback
402414
location = self.spec.get("location", "")
403415
if location:
404416
return str(Path(location) / path)
@@ -448,7 +460,11 @@ def get_url(self, path: str | PurePosixPath) -> str:
448460
elif self.protocol == "azure":
449461
return f"az://{full_path}"
450462
else:
451-
# Fallback: use protocol prefix
463+
from .storage_adapter import get_storage_adapter
464+
465+
adapter = get_storage_adapter(self.protocol)
466+
if adapter is not None:
467+
return adapter.get_url(self.spec, full_path)
452468
return f"{self.protocol}://{full_path}"
453469

454470
def put_file(self, local_path: str | Path, remote_path: str | PurePosixPath, metadata: dict | None = None) -> None:

tests/unit/test_storage_adapter.py

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,79 @@ def test_default_url_format(self):
105105
assert self.adapter.get_url({}, "data/file.dat") == "dummy://data/file.dat"
106106

107107

108+
from datajoint.storage import StorageBackend
109+
110+
111+
class _FakeFS:
112+
"""Minimal fake fsspec filesystem for testing."""
113+
protocol = "dummy"
114+
115+
116+
class _FSAdapter(StorageAdapter):
117+
"""Adapter that returns a fake filesystem."""
118+
119+
protocol = "testfs"
120+
required_keys = ("protocol",)
121+
allowed_keys = ("protocol",)
122+
123+
def create_filesystem(self, spec):
124+
return _FakeFS()
125+
126+
def get_url(self, spec, path):
127+
return f"https://test.example.com/{path}"
128+
129+
130+
class TestStorageBackendPluginDelegation:
131+
"""Tests for plugin delegation in StorageBackend methods."""
132+
133+
def setup_method(self):
134+
import datajoint.storage_adapter as sa_mod
135+
136+
sa_mod._adapter_registry["testfs"] = _FSAdapter()
137+
138+
def teardown_method(self):
139+
import datajoint.storage_adapter as sa_mod
140+
141+
sa_mod._adapter_registry.pop("testfs", None)
142+
143+
def test_create_filesystem_delegates_to_adapter(self):
144+
backend = StorageBackend.__new__(StorageBackend)
145+
backend.spec = {"protocol": "testfs"}
146+
backend.protocol = "testfs"
147+
backend._fs = None
148+
fs = backend._create_filesystem()
149+
assert isinstance(fs, _FakeFS)
150+
151+
def test_full_path_delegates_to_adapter(self):
152+
backend = StorageBackend.__new__(StorageBackend)
153+
backend.spec = {"protocol": "testfs", "location": "data"}
154+
backend.protocol = "testfs"
155+
result = backend._full_path("schema/ab/cd/hash123")
156+
assert result == "data/schema/ab/cd/hash123"
157+
158+
def test_full_path_empty_location(self):
159+
backend = StorageBackend.__new__(StorageBackend)
160+
backend.spec = {"protocol": "testfs", "location": ""}
161+
backend.protocol = "testfs"
162+
result = backend._full_path("schema/ab/cd/hash123")
163+
assert result == "schema/ab/cd/hash123"
164+
165+
def test_get_url_delegates_to_adapter(self):
166+
backend = StorageBackend.__new__(StorageBackend)
167+
backend.spec = {"protocol": "testfs", "location": ""}
168+
backend.protocol = "testfs"
169+
result = backend.get_url("schema/file.dat")
170+
assert result == "https://test.example.com/schema/file.dat"
171+
172+
def test_unsupported_protocol_error(self):
173+
backend = StorageBackend.__new__(StorageBackend)
174+
backend.spec = {"protocol": "totally_unknown_xyz"}
175+
backend.protocol = "totally_unknown_xyz"
176+
backend._fs = None
177+
with pytest.raises(DataJointError, match="Unsupported storage protocol"):
178+
backend._create_filesystem()
179+
180+
108181
import datajoint as dj
109182

110183

0 commit comments

Comments
 (0)