Use zarr3 and unpin dask (#486)

* use zarr3 and unpin dask

* relax ome-zarr upped bound

* allow-direct-references hatch

* set tool.hatch.metadata.allow-direct-references false

* WIP: support for zarr v3

* WIP: fix tests

* Remove storage of explicit bounding_box metadata

* Lint

* Pin snakemake

* Lint

* F-string issue

* Undo pin snakemake

* Update xenium morton code nb

* Updates

---------

Co-authored-by: Mark Keller <7525285+keller-mark@users.noreply.github.com>

Author: LucaMarconato

.python-version

@@ -1 +1 @@
-3.10
+3.12

demos/satija-2020/Snakefile

@@ -27,7 +27,9 @@ rule all:
 rule convert_to_zarr:
     input:
         cells_h5ad=(RAW_DIR / "uf-processed" / GLOBUS_ID / "cluster_marker_genes.h5ad"),
-        annotations_csv=(RAW_DIR / "annotations_spleen" / f"{GLOBUS_ID}.csv"),
+        # For some reasons f-strings in python 3.12 don't work as expected
+        # Reference: https://github.com/NeuralEnsemble/cobrawap/pull/105
+        annotations_csv=(RAW_DIR / "annotations_spleen" / str(GLOBUS_ID + ".csv")),
         cl_obo=(RAW_DIR / "cl.obo")
     output:
         cells=directory(PROCESSED_DIR / "satija_2020.h5ad.zarr"),
@@ -100,4 +102,4 @@ rule download_cl_obo:
     shell:
         '''
         curl -L -o {output} {params.file_url}
-        '''
+        '''

docs/data_options.rst

@@ -129,15 +129,18 @@ This is currently supported for the ``AnnDataWrapper`` class using the ``adata_s
     vc.widget()
 
 
-Or, with a Zarr store instance (instead of a local string path to a DirectoryStore):
+Or, with a Zarr store instance (instead of a local string path to a LocalStore/DirectoryStore):
 
 .. code-block:: python
 
     import zarr
+    from obstore.store import HTTPStore
     from vitessce import VitessceConfig, AnnDataWrapper
 
-    # ...
-    store = zarr.storage.FSStore("s3://my_bucket/path/to/my_store.adata.zarr")
+    obs_store = HTTPStore.from_url("https://my_bucket.example.com/path/to/my_store.adata.zarr")
+    store = zarr.storage.ObjectStore(obs_store, read_only=True)
+    # or
+    store = zarr.storage.LocalStore("./path/to/my_store.adata.zarr")
 
     vc = VitessceConfig(name="My Vitessce Configuration")
     vc.add_dataset(name="My Dataset").add_object(AnnDataWrapper(

docs/notebooks/spatial_data_blobs.ipynb

@@ -115,7 +115,7 @@
     ")\n",
     "# Add data to the configuration:\n",
     "wrapper = SpatialDataWrapper(\n",
-    "    sdata_path=spatialdata_filepath,\n",
+    "    sdata_store=spatialdata_filepath,\n",
     "    # The following paths are relative to the root of the SpatialData zarr store on-disk.\n",
     "    image_path=\"images/blobs_image\",\n",
     "    obs_segmentations_path=\"labels/blobs_labels\",\n",
@@ -129,7 +129,7 @@
     "    }\n",
     ")\n",
     "points_wrapper = SpatialDataWrapper(\n",
-    "    sdata_path=spatialdata_filepath,\n",
+    "    sdata_store=spatialdata_filepath,\n",
     "    # The following paths are relative to the root of the SpatialData zarr store on-disk.\n",
     "    obs_points_path=\"points/blobs_points\",\n",
     "    obs_feature_matrix_path=\"tables/table_points/X\", # TODO\n",
@@ -221,6 +221,27 @@
    "outputs": [],
    "source": []
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -245,7 +266,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.14"
+   "version": "3.12.5"
   }
  },
  "nbformat": 4,

docs/notebooks/spatial_data_xenium_morton.ipynb

@@ -47,7 +47,6 @@
     "from vitessce.data_utils import (\n",
     "    sdata_morton_sort_points,\n",
     "    sdata_points_process_columns,\n",
-    "    sdata_points_write_bounding_box_attrs,\n",
     "    sdata_points_modify_row_group_size,\n",
     "    sdata_morton_query_rect,\n",
     ")"
@@ -59,7 +58,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from spatialdata import read_zarr"
+    "from spatialdata import read_zarr\n",
+    "from spatialdata.models import PointsModel"
    ]
   },
   {
@@ -82,7 +82,7 @@
     "if not isdir(spatialdata_filepath):\n",
     "    if not isfile(zip_filepath):\n",
     "        os.makedirs(data_dir, exist_ok=True)\n",
-    "        urlretrieve('https://s3.embl.de/spatialdata/spatialdata-sandbox/xenium_rep1_io.zip', zip_filepath)\n",
+    "        urlretrieve('https://data-2.vitessce.io/sdata-datasets/xenium_rep1_io.spatialdata.zarr.zip', zip_filepath)\n",
     "    with zipfile.ZipFile(zip_filepath,\"r\") as zip_ref:\n",
     "        zip_ref.extractall(data_dir)\n",
     "        os.rename(join(data_dir, \"data.zarr\"), spatialdata_filepath)\n",
@@ -161,6 +161,15 @@
     "sdata.points['transcripts'].head()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "transformations = sdata['transcripts'].attrs['transform']"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -223,15 +232,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sdata[\"transcripts_with_morton_codes\"] = ddf\n",
-    "sdata.write_element(\"transcripts_with_morton_codes\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Step 4. Write bounding box metadata with `sdata_points_write_bounding_box_attrs`"
+    "del ddf.attrs['transform']"
    ]
   },
   {
@@ -240,14 +241,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sdata_points_write_bounding_box_attrs(sdata, \"transcripts_with_morton_codes\")"
+    "sdata[\"transcripts_with_morton_codes\"] = PointsModel.parse(ddf, feature_key=\"feature_name\", instance_key=\"cell_id\", transformations=transformations)\n",
+    "sdata.write_element(\"transcripts_with_morton_codes\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Step 5. Modify the row group sizes of the Parquet files with `sdata_points_modify_row_group_size`"
+    "### Step 4. Modify the row group sizes of the Parquet files with `sdata_points_modify_row_group_size`"
    ]
   },
   {
@@ -278,7 +280,7 @@
     "import pyarrow.parquet as pq\n",
     "from os.path import join\n",
     "\n",
-    "parquet_file = pq.ParquetFile(join(sdata.path, \"points\", \"transcripts_with_morton_codes\", \"points.parquet\", \"part.0.parquet\"))\n",
+    "parquet_file = pq.ParquetFile(join(sdata.path, \"points\", \"transcripts_with_morton_codes\", \"points.parquet\", \"part.1.parquet\"))\n",
     "\n",
     "# Get the number of row groups in this part-0 file.\n",
     "num_groups = parquet_file.num_row_groups\n",
@@ -308,29 +310,57 @@
     "wrapper = SpatialDataWrapper(\n",
     "    sdata_path=spatialdata_filepath,\n",
     "    # The following paths are relative to the root of the SpatialData zarr store on-disk.\n",
-    "    image_path=\"images/rasterized\",\n",
+    "    image_path=\"images/morphology_focus\",\n",
     "    table_path=\"tables/table\",\n",
     "    obs_feature_matrix_path=\"tables/table/X\",\n",
-    "    obs_spots_path=\"shapes/cells\",\n",
+    "    #obs_spots_path=\"shapes/cell_circles\",\n",
+    "    obs_segmentations_path=\"shapes/cell_boundaries\",\n",
     "    coordinate_system=\"global\",\n",
     "    coordination_values={\n",
     "        # The following tells Vitessce to consider each observation as a \"spot\"\n",
     "        \"obsType\": \"cell\",\n",
     "    }\n",
     ")\n",
-    "dataset = vc.add_dataset(name='MERFISH').add_object(wrapper)\n",
+    "points_wrapper = SpatialDataWrapper(\n",
+    "    sdata_path=spatialdata_filepath,\n",
+    "    # The following paths are relative to the root of the SpatialData zarr store on-disk.\n",
+    "    obs_points_path=\"points/transcripts_with_morton_codes\",\n",
+    "    obs_feature_matrix_path=\"tables/dense_table/X\",\n",
+    "    coordinate_system=\"global\",\n",
+    "    coordination_values={\n",
+    "        \"obsType\": \"point\",\n",
+    "        \"featureType\": \"gene\",\n",
+    "    }\n",
+    ")\n",
+    "dataset = vc.add_dataset(name='MERFISH').add_object(wrapper).add_object(points_wrapper)\n",
     "\n",
     "# Add views (visualizations) to the configuration:\n",
     "spatial = vc.add_view(\"spatialBeta\", dataset=dataset)\n",
     "feature_list = vc.add_view(\"featureList\", dataset=dataset)\n",
     "layer_controller = vc.add_view(\"layerControllerBeta\", dataset=dataset)\n",
     "obs_sets = vc.add_view(\"obsSets\", dataset=dataset)\n",
     "\n",
+    "\"\"\"\n",
     "vc.link_views_by_dict([spatial, layer_controller], {\n",
     "    'spotLayer': CL([{\n",
     "        'obsType': 'cell',\n",
     "    }]),\n",
     "}, scope_prefix=get_initial_coordination_scope_prefix(\"A\", \"obsSpots\"))\n",
+    "\"\"\"\n",
+    "\n",
+    "vc.link_views_by_dict([spatial, layer_controller], {\n",
+    "    'segmentationLayer': CL([{\n",
+    "        'segmentationChannel': CL([{\n",
+    "            'obsType': 'cell',\n",
+    "        }]),\n",
+    "    }]),\n",
+    "}, scope_prefix=get_initial_coordination_scope_prefix(\"A\", \"obsSegmentations\"))\n",
+    "\n",
+    "vc.link_views_by_dict([spatial, layer_controller], {\n",
+    "    'pointLayer': CL([{\n",
+    "        'obsType': 'point',\n",
+    "    }]),\n",
+    "}, scope_prefix=get_initial_coordination_scope_prefix(\"A\", \"obsPoints\"))\n",
     "\n",
     "vc.link_views([spatial, layer_controller, feature_list, obs_sets], ['obsType'], [wrapper.obs_type_label])\n",
     "\n",
@@ -379,7 +409,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.14"
+   "version": "3.12.5"
   }
  },
  "nbformat": 4,

pyproject.toml

@@ -4,14 +4,14 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vitessce"
-version = "3.8.4"
+version = "3.9.0"
 authors = [
   { name="Mark Keller", email="mark_keller@hms.harvard.edu" },
 ]
 description = "Jupyter widget facilitating interactive visualization of spatial single-cell data with Vitessce"
 readme = "README.md"
 license = {file = "LICENSE"}
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 keywords = ["ipython", "jupyter", "widgets"]
 classifiers = [
   'Development Status :: 4 - Beta',
@@ -29,8 +29,7 @@ dependencies = [
   'pandas>=1.1.2',
   'black>=21.11b1',
   'numpy>=1.21.2',
-  'zarr>=2.5.0,<3',
-  'numcodecs>=0.5.7,<0.16.0',
+  'zarr>=3.0.0',
 ]
 
 [project.optional-dependencies]
@@ -61,22 +60,22 @@ docs = [
 all = [
   # For data_utils
   'negspy>=0.2.24',
-  'anndata>=0.7.8',
+  'anndata>=0.12.10',
   # scanpy < 1.10.3 does not support numpy >= 2.0.0 and does not
   # Reference: https://github.com/scverse/scanpy/pull/3115/files
   'scanpy>=1.10.2',
-  'ome-zarr<0.10.3',
+  'ome-zarr>=0.12.2',
   'tifffile>=2020.10.1',
 
   'jupyter-server-proxy>=1.5.2',
   'oxc-py>=0.1.1',
-  'anywidget>=0.9.10',
+  'anywidget>=0.11.0',
   'uvicorn>=0.17.0',
   'ujson>=4.0.1',
   'starlette>=0.14.0',
   'generate-tiff-offsets>=0.1.9',
   'kerchunk>=0.2.6',
-  'fsspec',
+  'obstore',
 
   # aiofiles is not explicitly referenced in our code,
   # but it is an implicit dependency of starlette==0.14.0.
@@ -88,8 +87,7 @@ building = []
 testing = []
 linting = []
 notebook = [
-  'spatialdata>=0.3.0',
-  'dask[dataframe]==2024.11.1',
+  'spatialdata>=0.7.3',
   'marimo',
   'starlette>=0.42.0',
   'tqdm>=4.1.0',
@@ -121,8 +119,7 @@ dev = [
   'boto3>=1.16.30',
   'scikit-misc>=0.1.3',
   'autopep8>=2.0.2',
-  'spatialdata>=0.3.0',
-  'dask[dataframe]==2024.11.1',
+  'spatialdata>=0.7.3',
 ]
 
 [tool.uv]

src/vitessce/data_utils/__init__.py

@@ -22,9 +22,9 @@
     sdata_morton_sort_points,
     # Other helper functions
     sdata_points_process_columns,
-    sdata_points_write_bounding_box_attrs,
     sdata_points_modify_row_group_size,
     # Functions for querying
     sdata_morton_query_rect,
     row_ranges_to_row_indices,
+    MORTON_CODE_EXTREME_VALUE_INDICATOR,
 )

src/vitessce/data_utils/entities.py

@@ -196,8 +196,6 @@ def __init__(self, f, profile_paths, assembly='hg38', starting_resolution=5000,
 
         num_profiles = len(profile_paths)
 
-        compressor = 'default'
-
         chromosomes = [str(chr_name) for chr_name in nc.get_chromorder(
             assembly)[:25]]  # TODO: should more than chr1-chrM be used?
         chroms_length_arr = np.array(
@@ -217,8 +215,8 @@ def __init__(self, f, profile_paths, assembly='hg38', starting_resolution=5000,
             # Create each resolution group.
             for resolution in resolutions:
                 chr_shape = (num_profiles, math.ceil(chr_len / resolution))
-                chr_group.create_dataset(str(
-                    resolution), shape=chr_shape, dtype="f4", fill_value=np.nan, compressor=compressor)
+                chr_group.create_array(str(
+                    resolution), shape=chr_shape, dtype="f4", fill_value=np.nan)
 
         # f.attrs should contain the properties required for HiGlass's "tileset_info" requests.
         f.attrs['row_infos'] = [