Skip to content

Commit c0ca91a

Browse files
dimitri-yatsenkodimitri-yatsenko
authored
Merge pull request #1 from d-v-b/feat/tests
feat: add tests and refactor codec
2 parents 31a92a1 + 176c12a commit c0ca91a

9 files changed

Lines changed: 393 additions & 81 deletions

File tree

‎.github/workflows/tests.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Tests
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
workflow_dispatch:
9+
10+
jobs:
11+
test:
12+
runs-on: ubuntu-latest
13+
strategy:
14+
matrix:
15+
python-version: ["3.11", "3.12", "3.13"]
16+
17+
steps:
18+
- uses: actions/checkout@v4
19+
20+
- name: Install uv
21+
uses: astral-sh/setup-uv@v4
22+
23+
- name: Set up Python ${{ matrix.python-version }}
24+
run: uv python install ${{ matrix.python-version }}
25+
26+
- name: Install dependencies
27+
run: uv sync --dev
28+
29+
- name: Run tests
30+
run: uv run pytest tests -v

‎.gitignore‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# C extensions
7+
*.so
8+
9+
# Distribution / packaging
10+
.Python
11+
build/
12+
develop-eggs/
13+
dist/
14+
downloads/
15+
eggs/
16+
.eggs/
17+
lib/
18+
lib64/
19+
parts/
20+
sdist/
21+
var/
22+
wheels/
23+
share/python-wheels/
24+
*.egg-info/
25+
.installed.cfg
26+
*.egg
27+
MANIFEST
28+
29+
# PyInstaller
30+
*.manifest
31+
*.spec
32+
33+
# Installer logs
34+
pip-log.txt
35+
pip-delete-this-directory.txt
36+
37+
# Unit test / coverage reports
38+
htmlcov/
39+
.tox/
40+
.nox/
41+
.coverage
42+
.coverage.*
43+
.cache
44+
nosetests.xml
45+
coverage.xml
46+
*.cover
47+
*.py,cover
48+
.hypothesis/
49+
.pytest_cache/
50+
51+
# Translations
52+
*.mo
53+
*.pot
54+
55+
# Django stuff:
56+
*.log
57+
local_settings.py
58+
db.sqlite3
59+
db.sqlite3-journal
60+
61+
# Flask stuff:
62+
instance/
63+
.webassets-cache
64+
65+
# Scrapy stuff:
66+
.scrapy
67+
68+
# Sphinx documentation
69+
docs/_build/
70+
71+
# PyBuilder
72+
.pybuilder/
73+
target/
74+
75+
# Jupyter Notebook
76+
.ipynb_checkpoints
77+
78+
# IPython
79+
profile_default/
80+
ipython_config.py
81+
82+
# pipenv
83+
Pipfile.lock
84+
85+
# PEP 582
86+
__pypackages__/
87+
88+
# Celery stuff
89+
celerybeat-schedule
90+
celerybeat.pid
91+
92+
# SageMath parsed files
93+
*.sage.py
94+
95+
# Environments
96+
.env
97+
.venv
98+
env/
99+
venv/
100+
ENV/
101+
env.bak/
102+
venv.bak/
103+
104+
# Spyder project settings
105+
.spyderproject
106+
.spyproject
107+
108+
# Rope project settings
109+
.ropeproject
110+
111+
# mkdocs documentation
112+
/site
113+
114+
# mypy
115+
.mypy_cache/
116+
.dmypy.json
117+
dmypy.json
118+
119+
# Pyre type checker
120+
.pyre/
121+
122+
# pytype static type analyzer
123+
.pytype/
124+
125+
# Cython debug symbols
126+
cython_debug/
127+
128+
# IDE
129+
.idea/
130+
.vscode/
131+
*.swp
132+
*.swo
133+
*~
134+
135+
# OS
136+
.DS_Store
137+
Thumbs.db

‎.python-version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.11

‎pyproject.toml‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name = "dj-zarr-codecs"
33
version = "0.1.0"
44
description = "DataJoint codecs for Zarr array storage"
55
readme = "README.md"
6-
requires-python = ">=3.10"
6+
requires-python = ">=3.11"
77
license = {text = "MIT"}
88
authors = [
99
{name = "Davis Bennett", email = "davis.v.bennett@gmail.com"},
@@ -24,14 +24,15 @@ classifiers = [
2424
dependencies = [
2525
"datajoint>=2.0.0a22",
2626
"zarr>=2.0",
27-
"numpy>=1.20",
27+
"numpy>=2.0",
2828
]
2929

30-
[project.optional-dependencies]
30+
[dependency-groups]
3131
dev = [
3232
"pytest>=7.0",
3333
"pytest-cov>=4.0",
3434
"ruff>=0.1.0",
35+
"testcontainers[mysql]>=4.14.0",
3536
]
3637

3738
[project.urls]
@@ -47,12 +48,14 @@ zarr = "dj_zarr_codecs:ZarrCodec"
4748
requires = ["setuptools>=61.0"]
4849
build-backend = "setuptools.build_meta"
4950

51+
[tool.uv.sources]
52+
datajoint = {git = "https://github.com/datajoint/datajoint-python.git"}
53+
5054
[tool.setuptools.packages.find]
5155
where = ["src"]
5256

5357
[tool.pytest.ini_options]
5458
testpaths = ["tests"]
55-
addopts = "-v --cov=dj_zarr_codecs --cov-report=term-missing"
5659

5760
[tool.ruff]
5861
line-length = 100

‎src/dj_zarr_codecs/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"""DataJoint codecs for Zarr array storage."""
22

3-
from .codecs import ZarrCodec
3+
from .codecs import ZarrArrayCodec
44

55
__version__ = "0.1.0"
6-
__all__ = ["ZarrCodec"]
6+
__all__ = ["ZarrArrayCodec"]

‎src/dj_zarr_codecs/codecs.py‎

Lines changed: 43 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,11 @@
66

77
import numpy as np
88
import zarr
9+
import datajoint as dj
10+
from datajoint import DataJointError
11+
from datajoint.builtin_codecs import SchemaCodec
912

10-
try:
11-
import datajoint as dj
12-
from datajoint import DataJointError
13-
from datajoint.builtin_codecs import SchemaCodec
14-
except ImportError as e:
15-
raise ImportError(
16-
"datajoint>=2.0.0a22 is required. Install with: pip install 'datajoint>=2.0.0a22'"
17-
) from e
18-
19-
20-
class ZarrCodec(SchemaCodec):
13+
class ZarrArrayCodec(SchemaCodec):
2114
"""
2215
Store numpy arrays in Zarr format with schema-addressed paths.
2316
@@ -78,7 +71,6 @@ class Recording(dj.Manual):
7871
"""
7972

8073
name = "zarr"
81-
CODEC_VERSION = "1.0" # Data format version for backward compatibility
8274

8375
def validate(self, value: Any) -> None:
8476
"""
@@ -94,16 +86,16 @@ def validate(self, value: Any) -> None:
9486
DataJointError
9587
If value is not a numpy array or has object dtype.
9688
"""
97-
if not isinstance(value, np.ndarray):
98-
raise DataJointError(
99-
f"<zarr> requires numpy.ndarray, got {type(value).__name__}"
89+
if not isinstance(value, np.ndarray | zarr.Array):
90+
raise TypeError(
91+
f"<zarr> requires a Numpy array or Zarr array, got {type(value).__name__}"
10092
)
10193
if value.dtype == object:
10294
raise DataJointError("<zarr> does not support object dtype arrays")
10395

10496
def encode(
10597
self,
106-
value: np.ndarray,
98+
value: np.ndarray | zarr.Array,
10799
*,
108100
key: dict | None = None,
109101
store_name: str | None = None,
@@ -130,40 +122,35 @@ def encode(
130122
DataJointError
131123
If encoding fails.
132124
"""
133-
try:
134-
# Extract context from key
135-
schema, table, field, primary_key = self._extract_context(key)
136-
137-
# Build schema-addressed path
138-
path, _token = self._build_path(
139-
schema, table, field, primary_key, ext=".zarr", store_name=store_name
140-
)
141-
142-
# Get storage backend
143-
backend = self._get_backend(store_name)
144-
145-
# Get fsspec mapper for direct Zarr write
146-
store_map = backend.get_fsmap(path)
147-
148-
# Write array to Zarr format
149-
zarr.save_array(store_map, value)
150-
151-
# Store version metadata in Zarr attributes
152-
z = zarr.open(store_map, mode="r+")
153-
z.attrs["codec_version"] = self.CODEC_VERSION
154-
z.attrs["codec_name"] = self.name
155-
156-
# Return metadata for database storage
157-
return {
158-
"path": path,
159-
"store": store_name,
160-
"codec_version": self.CODEC_VERSION,
161-
"shape": list(value.shape),
162-
"dtype": str(value.dtype),
163-
}
164-
165-
except Exception as e:
166-
raise DataJointError(f"Failed to encode Zarr array: {e}") from e
125+
# import here to avoid circular import
126+
from dj_zarr_codecs import __version__
127+
# Extract context from key
128+
schema, table, field, primary_key = self._extract_context(key)
129+
130+
# Build schema-addressed path
131+
path, _token = self._build_path(
132+
schema, table, field, primary_key, ext=".zarr", store_name=store_name
133+
)
134+
135+
# Get storage backend
136+
backend = self._get_backend(store_name)
137+
138+
# Get fsspec mapper for direct Zarr write
139+
store_map = backend.get_fsmap(path)
140+
141+
zarr.create_array(store=store_map, data=value, write_data=True)
142+
143+
# Return metadata for database storage (stored as JSON column)
144+
return {
145+
"path": path,
146+
"store": store_name,
147+
"shape": list(value.shape),
148+
"dtype": str(value.dtype),
149+
"provenance": {
150+
"datajoint-python": dj.__version__,
151+
"dj-zarr-codecs": __version__,
152+
},
153+
}
167154

168155
def decode(self, stored: dict, *, key: dict | None = None) -> zarr.Array:
169156
"""
@@ -187,30 +174,11 @@ def decode(self, stored: dict, *, key: dict | None = None) -> zarr.Array:
187174
DataJointError
188175
If decoding fails.
189176
"""
190-
try:
191-
# Get storage backend
192-
backend = self._get_backend(stored.get("store"))
177+
# Get storage backend
178+
backend = self._get_backend(stored.get("store"))
193179

194-
# Get fsspec mapper for Zarr path
195-
store_map = backend.get_fsmap(stored["path"])
196-
197-
# Open Zarr array (read-only)
198-
z = zarr.open(store_map, mode="r")
199-
200-
# Check codec version for backward compatibility
201-
# Priority: Zarr attrs > DB metadata > default "1.0"
202-
version = z.attrs.get(
203-
"codec_version", stored.get("codec_version", "1.0")
204-
)
180+
# Get fsspec mapper for Zarr path
181+
store_map = backend.get_fsmap(stored["path"])
205182

206-
# All v1.x versions are compatible
207-
if version.startswith("1."):
208-
return z
209-
else:
210-
raise DataJointError(
211-
f"Unsupported zarr codec version: {version}. "
212-
f"Upgrade dj-zarr-codecs or migrate data."
213-
)
214-
215-
except Exception as e:
216-
raise DataJointError(f"Failed to decode Zarr array: {e}") from e
183+
# Open Zarr array (read-only)
184+
return zarr.open(store_map, mode="r")

‎tests/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)