make v3 regression testing for string work properly

Author: d-v-b

tests/test_dtype/test_npy/test_string.py

@@ -19,7 +19,7 @@ class TestVariableLengthString(BaseTestZDType):
             np.dtype("|S10"),
         )
         valid_json_v2 = ({"name": "|O", "object_codec_id": "vlen-utf8"},)
-        valid_json_v3 = ("variable_length_utf8",)
+        valid_json_v3 = ("string",)
         invalid_json_v2 = (
             "|S10",
             "|f8",
@@ -53,7 +53,7 @@ class TestVariableLengthString(BaseTestZDType):  # type: ignore[no-redef]
             np.dtype("|S10"),
         )
         valid_json_v2 = ({"name": "|O", "object_codec_id": "vlen-utf8"},)
-        valid_json_v3 = ("variable_length_utf8",)
+        valid_json_v3 = ("string",)
         invalid_json_v2 = (
             "|S10",
             "|f8",

tests/test_regression/scripts/v3.0.8.py

@@ -5,6 +5,53 @@
 # ]
 # ///
 
+# /// script
+# requires-python = ">=3.11"
+# dependencies = [
+#  "zarr==2.18",
+#  "numcodecs==0.15"
+# ]
+# ///
+
+import argparse
+
+import zarr
+from zarr.abc.store import Store
+
+def copy_group(
+    *, node: zarr.Group, store: Store, path: str, overwrite: bool
+) -> zarr.Group:
+    result = zarr.create_group(
+        store=store,
+        path=path,
+        overwrite=overwrite,
+        attributes=node.attrs.asdict(),
+        zarr_format=node.metadata.zarr_format)
+    for key, child in node.members():
+        child_path = f"{path}/{key}"
+        if isinstance(child, zarr.Group):
+            copy_group(node=child, store=store, path=child_path, overwrite=overwrite)
+        else:
+            copy_array(node=child, store=store, overwrite=overwrite, path=child_path)
+    return result
+
+
+def copy_array(
+    *, node: zarr.Array, store: Store, path: str, overwrite: bool
+) -> zarr.Array:
+    result = zarr.from_array(store, name=path, data=node, write_data=True)
+    return result
+
+
+def copy_node(
+    node: zarr.Group | zarr.Array, store: Store, path: str, overwrite: bool
+) -> zarr.Group | zarr.Array:
+    if isinstance(node, zarr.Group):
+        return copy_group(node=node, store=store, path=path, overwrite=overwrite)
+    else:
+        return copy_array(node=node, store=store, path=path, overwrite=overwrite)
+
+
 def cli() -> None:
     parser = argparse.ArgumentParser(
         description="Copy a zarr hierarchy from one location to another"
@@ -15,7 +62,7 @@ def cli() -> None:
 
     src, dst = args.source, args.destination
     root_src = zarr.open(src, mode="r")
-    result = copy_node(node=root_src, store=zarr.NestedDirectoryStore(dst), path="", overwrite=True)
+    result = copy_node(node=root_src, store=dst, path="", overwrite=True)
 
     print(f"successfully created {result} at {dst}")
 

tests/test_regression/test_v2_dtype_regression.py

@@ -10,6 +10,9 @@
 from numcodecs import LZ4, LZMA, Blosc, GZip, VLenBytes, VLenUTF8, Zstd
 
 import zarr
+import zarr.abc
+import zarr.abc.codec
+import zarr.codecs as zarrcodecs
 from zarr.core.array import Array
 from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding
 from zarr.core.dtype.npy.bytes import VariableLengthBytes
@@ -38,6 +41,7 @@ class ArrayParams:
     values: np.ndarray[tuple[int], np.dtype[np.generic]]
     fill_value: np.generic | str | int | bytes
     filters: tuple[numcodecs.abc.Codec, ...] = ()
+    serializer: str | None = None
     compressor: numcodecs.abc.Codec
 
 
@@ -77,15 +81,15 @@ class ArrayParams:
     ArrayParams(
         values=np.array(["a", "bb", "ccc", "dddd"], dtype="O"),
         fill_value="1",
-        filters=(VLenUTF8(),),
+        serializer="vlen-utf8",
         compressor=GZip(),
     )
 ]
 vlen_bytes_cases = [
     ArrayParams(
         values=np.array([b"a", b"bb", b"ccc", b"dddd"], dtype="O"),
         fill_value=b"1",
-        filters=(VLenBytes(),),
+        serializer="vlen-bytes",
         compressor=GZip(),
     )
 ]
@@ -102,7 +106,7 @@ class ArrayParams:
 
 
 @pytest.fixture
-def source_array(tmp_path: Path, request: pytest.FixtureRequest) -> Array:
+def source_array_v2(tmp_path: Path, request: pytest.FixtureRequest) -> Array:
     """
     Writes a zarr array to a temporary directory based on the provided ArrayParams. The array is
     returned.
@@ -113,19 +117,22 @@ def source_array(tmp_path: Path, request: pytest.FixtureRequest) -> Array:
     compressor = array_params.compressor
     chunk_key_encoding = V2ChunkKeyEncoding(separator="/")
     dtype: ZDTypeLike
-    if array_params.values.dtype == np.dtype("|O") and array_params.filters == (VLenUTF8(),):
+    if array_params.values.dtype == np.dtype("|O") and array_params.serializer == "vlen-utf8":
         dtype = VariableLengthUTF8()  # type: ignore[assignment]
-    elif array_params.values.dtype == np.dtype("|O") and array_params.filters == (VLenBytes(),):
+        filters = array_params.filters + (VLenUTF8(),)
+    elif array_params.values.dtype == np.dtype("|O") and array_params.serializer == "vlen-bytes":
         dtype = VariableLengthBytes()
+        filters = array_params.filters + (VLenBytes(),)
     else:
         dtype = array_params.values.dtype
+        filters = array_params.filters
     z = zarr.create_array(
         store,
         shape=array_params.values.shape,
         dtype=dtype,
         chunks=array_params.values.shape,
         compressors=compressor,
-        filters=array_params.filters,
+        filters=filters,
         fill_value=array_params.fill_value,
         order="C",
         chunk_key_encoding=chunk_key_encoding,
@@ -136,29 +143,100 @@ def source_array(tmp_path: Path, request: pytest.FixtureRequest) -> Array:
     return z
 
 
+@pytest.fixture
+def source_array_v3(tmp_path: Path, request: pytest.FixtureRequest) -> Array:
+    """
+    Writes a zarr array to a temporary directory based on the provided ArrayParams. The array is
+    returned.
+    """
+    dest = tmp_path / "in"
+    store = LocalStore(dest)
+    array_params: ArrayParams = request.param
+    chunk_key_encoding = V2ChunkKeyEncoding(separator="/")
+    dtype: ZDTypeLike
+    serializer: Literal["auto"] | zarr.abc.codec.Codec
+    if array_params.values.dtype == np.dtype("|O") and array_params.serializer == "vlen-utf8":
+        dtype = VariableLengthUTF8()  # type: ignore[assignment]
+        serializer = zarrcodecs.VLenUTF8Codec()
+    elif array_params.values.dtype == np.dtype("|O") and array_params.serializer == "vlen-bytes":
+        dtype = VariableLengthBytes()
+        serializer = zarrcodecs.VLenBytesCodec()
+    else:
+        dtype = array_params.values.dtype
+        serializer = "auto"
+    if array_params.compressor == GZip():
+        compressor = zarrcodecs.GzipCodec()
+    else:
+        msg = (
+            "This test is only compatible with gzip compression at the moment, because the author"
+            "did not want to implement a complete abstraction layer for v2 and v3 codecs in this test."
+        )
+        raise ValueError(msg)
+    z = zarr.create_array(
+        store,
+        shape=array_params.values.shape,
+        dtype=dtype,
+        chunks=array_params.values.shape,
+        compressors=compressor,
+        filters=array_params.filters,
+        serializer=serializer,
+        fill_value=array_params.fill_value,
+        chunk_key_encoding=chunk_key_encoding,
+        write_data=True,
+        zarr_format=3,
+    )
+    z[:] = array_params.values
+    return z
+
+
 # TODO: make this dynamic based on the installed scripts
 script_paths = [Path(__file__).resolve().parent / "scripts" / "v2.18.py"]
 
 
 @pytest.mark.skipif(not runner_installed(), reason="no python script runner installed")
 @pytest.mark.parametrize(
-    "source_array", array_cases_v2_18, indirect=True, ids=tuple(map(str, array_cases_v2_18))
+    "source_array_v2", array_cases_v2_18, indirect=True, ids=tuple(map(str, array_cases_v2_18))
 )
 @pytest.mark.parametrize("script_path", script_paths)
-def test_roundtrip_v2(source_array: Array, tmp_path: Path, script_path: Path) -> None:
+def test_roundtrip_v2(source_array_v2: Array, tmp_path: Path, script_path: Path) -> None:
     out_path = tmp_path / "out"
     copy_op = subprocess.run(
         [
             "uv",
             "run",
-            script_path,
-            str(source_array.store).removeprefix("file://"),
+            str(script_path),
+            str(source_array_v2.store).removeprefix("file://"),
             str(out_path),
         ],
         capture_output=True,
         text=True,
     )
     assert copy_op.returncode == 0
     out_array = zarr.open_array(store=out_path, mode="r", zarr_format=2)
-    assert source_array.metadata.to_dict() == out_array.metadata.to_dict()
-    assert np.array_equal(source_array[:], out_array[:])
+    assert source_array_v2.metadata.to_dict() == out_array.metadata.to_dict()
+    assert np.array_equal(source_array_v2[:], out_array[:])
+
+
+@pytest.mark.skipif(not runner_installed(), reason="no python script runner installed")
+@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning")
+@pytest.mark.parametrize(
+    "source_array_v3", array_cases_v3_08, indirect=True, ids=tuple(map(str, array_cases_v3_08))
+)
+def test_roundtrip_v3(source_array_v3: Array, tmp_path: Path) -> None:
+    script_path = Path(__file__).resolve().parent / "scripts" / "v3.0.8.py"
+    out_path = tmp_path / "out"
+    copy_op = subprocess.run(
+        [
+            "uv",
+            "run",
+            str(script_path),
+            str(source_array_v3.store).removeprefix("file://"),
+            str(out_path),
+        ],
+        capture_output=True,
+        text=True,
+    )
+    assert copy_op.returncode == 0
+    out_array = zarr.open_array(store=out_path, mode="r", zarr_format=3)
+    assert source_array_v3.metadata.to_dict() == out_array.metadata.to_dict()
+    assert np.array_equal(source_array_v3[:], out_array[:])