Merge branch 'main' of https://github.com/eopf-explorer/data-model into chore/src-layout

Author: d-v-b

.gitignore

@@ -206,4 +206,7 @@ marimo/_static/
 marimo/_lsp/
 __marimo__/
 
-tests-output/
+tests-output/
+
+# uv
+uv.lock

.vscode/launch.json

@@ -16,8 +16,8 @@
                 "https://objectstore.eodc.eu:2222/e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr",
                 "./tests-output/eopf_geozarr/s2b_test.zarr",
                 "--groups", "/measurements/reflectance/r10m", "/measurements/reflectance/r20m", "/measurements/reflectance/r60m", "/quality/l1c_quicklook/r10m",
-                "--spatial-chunk", "512",
-                "--min-dimension", "128",
+                "--spatial-chunk", "4096",
+                "--min-dimension", "256",
                 "--tile-width", "256",
                 "--max-retries", "2",
                 "--verbose"
@@ -38,25 +38,77 @@
             "args": [
                 "convert",
                 "https://objectstore.eodc.eu:2222/e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr",
-                "s3://esa-zarr-sentinel-explorer/tests-output/eopf_geozarr/s2b_test.zarr",
+                "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/s2b_test.zarr",
                 "--groups", "/measurements/reflectance/r10m", "/measurements/reflectance/r20m", "/measurements/reflectance/r60m", "/quality/l1c_quicklook/r10m",
                 "--spatial-chunk", "4096",
                 "--min-dimension", "256",
                 "--tile-width", "256",
                 "--max-retries", "2",
+                "--dask-cluster",
+                "--verbose"
+            ],
+            "cwd": "${workspaceFolder}",
+            "justMyCode": false,
+            "console": "integratedTerminal",
+            "env": {
+                "PYTHONPATH": "${workspaceFolder}/.venv/bin",
+                "AWS_PROFILE": "eopf-explorer",
+                "AWS_DEFAULT_REGION": "de",
+                "AWS_S3_ENDPOINT": "https://s3.de.io.cloud.ovh.net/"
+            },
+
+        },
+        {
+            // eopf_geozarr convert https://objectstore.eodc.eu:2222/e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr /tmp/tmp7mmjkjk3/s2b_subset_test.zarr --groups /measurements/reflectance/r10m --spatial-chunk 512 --min-dimension 128 --tile-width 256 --max-retries 2 --verbose
+            "name": "Convert to GeoZarr S2L2A (S3)",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "eopf_geozarr",
+            "args": [
+                "convert",
+                "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202507-s02msil2a/04/products/cpm_v256/S2A_MSIL2A_20250704T094051_N0511_R036_T33SWB_20250704T115824.zarr",
+                "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/S2A_MSIL2A_20250704T094051_N0511_R036_T33SWB_20250704T115824.zarr",
+                "--groups", "/measurements/reflectance/r10m", "/measurements/reflectance/r20m", "/measurements/reflectance/r60m", "/quality/l2a_quicklook/r10m",
+                "--spatial-chunk", "1024",
+                "--min-dimension", "256",
+                "--tile-width", "256",
+                "--max-retries", "2",
+                "--dask-cluster",
                 "--verbose"
             ],
             "cwd": "${workspaceFolder}",
             "justMyCode": false,
             "console": "integratedTerminal",
             "env": {
                 "PYTHONPATH": "${workspaceFolder}/.venv/bin",
-                "AWS_ACCESS_KEY_ID": "secret",
-                "AWS_SECRET_ACCESS_KEY": "secret",
-                "AWS_DEFAULT_REGION": "gra",
-                "AWS_S3_ENDPOINT": "https://s3.gra.io.cloud.ovh.net/" 
+                "AWS_PROFILE": "eopf-explorer",
+                "AWS_DEFAULT_REGION": "de",
+                "AWS_S3_ENDPOINT": "https://s3.de.io.cloud.ovh.net/"
             },
-            
+
+        },
+        {
+            // eopf_geozarr convert https://objectstore.eodc.eu:2222/e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr /tmp/tmp7mmjkjk3/s2b_subset_test.zarr --groups /measurements/reflectance/r10m --spatial-chunk 512 --min-dimension 128 --tile-width 256 --max-retries 2 --verbose
+            "name": "Info GeoZarr S2L2A (S3)",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "eopf_geozarr",
+            "args": [
+                "info",
+                "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/S2A_MSIL2A_20250704T094051_N0511_R036_T33SWB_20250704T115824.zarr",
+                "--verbose",
+                "--html-output", "dataset_info.html"
+            ],
+            "cwd": "${workspaceFolder}",
+            "justMyCode": false,
+            "console": "integratedTerminal",
+            "env": {
+                "PYTHONPATH": "${workspaceFolder}/.venv/bin",
+                "AWS_PROFILE": "eopf-explorer",
+                "AWS_DEFAULT_REGION": "de",
+                "AWS_S3_ENDPOINT": "https://s3.de.io.cloud.ovh.net/"
+            },
+
         }
     ]
 }

README.md

@@ -14,6 +14,8 @@ This library provides tools to convert EOPF datasets to GeoZarr-spec 0.4 complia
 - **CF Conventions**: Proper CF standard names and grid_mapping attributes
 - **Robust Processing**: Band-by-band writing with validation and retry logic
 - **S3 Support**: Direct output to Amazon S3 buckets with automatic credential validation
+- **Parallel Processing**: Optional dask cluster support for parallel chunk processing
+- **Chunk Alignment**: Automatic chunk alignment to prevent data corruption with dask
 
 ## GeoZarr Compliance Features
 
@@ -51,6 +53,12 @@ eopf-geozarr convert input.zarr output.zarr
 # Convert EOPF dataset to GeoZarr format (S3 output)
 eopf-geozarr convert input.zarr s3://my-bucket/path/to/output.zarr
 
+# Convert with parallel processing using dask cluster
+eopf-geozarr convert input.zarr output.zarr --dask-cluster
+
+# Convert with dask cluster and verbose output
+eopf-geozarr convert input.zarr output.zarr --dask-cluster --verbose
+
 # Get information about a dataset
 eopf-geozarr info input.zarr
 
@@ -111,6 +119,42 @@ aws configure
 - **Error Handling**: Provides helpful error messages for S3 configuration issues
 - **Performance**: Optimized for S3 with proper chunking and retry logic
 
+### Parallel Processing with Dask
+
+The library supports parallel processing using dask clusters for improved performance on large datasets:
+
+```bash
+# Enable dask cluster for parallel processing
+eopf-geozarr convert input.zarr output.zarr --dask-cluster
+
+# With verbose output to see cluster information
+eopf-geozarr convert input.zarr output.zarr --dask-cluster --verbose
+```
+
+#### Dask Features
+
+- **Local Cluster**: Automatically starts a local dask cluster with multiple workers
+- **Dashboard Access**: Provides access to the dask dashboard for monitoring (shown in verbose mode)
+- **Automatic Cleanup**: Properly closes the cluster even if errors occur during processing
+- **Chunk Alignment**: Automatically aligns Zarr chunks with dask chunks to prevent data corruption
+- **Memory Efficiency**: Better memory management through parallel chunk processing
+- **Error Handling**: Graceful handling of dask import errors with helpful installation instructions
+
+#### Chunk Alignment
+
+The library includes advanced chunk alignment logic to prevent the common issue of overlapping chunks when using dask:
+
+- **Smart Detection**: Automatically detects if data is dask-backed and uses existing chunk structure
+- **Aligned Calculation**: Uses `calculate_aligned_chunk_size()` to find optimal chunk sizes that divide evenly into data dimensions
+- **Proper Rechunking**: Ensures datasets are rechunked to match encoding before writing
+- **Fallback Logic**: For non-dask arrays, uses reasonable chunk sizes that don't exceed data dimensions
+
+This prevents errors like:
+```
+❌ Failed to write tci after 2 attempts: Specified Zarr chunks encoding['chunks']=(1, 3660, 3660)
+for variable named 'tci' would overlap multiple Dask chunks
+```
+
 #### S3 Python API
 
 ```python
@@ -202,7 +246,22 @@ Downsample a 2D array using block averaging.
 
 #### `calculate_aligned_chunk_size`
 
-Calculate a chunk size that aligns well with the data dimension.
+Calculate a chunk size that divides evenly into the dimension size. This ensures that Zarr chunks align properly with the data dimensions, preventing chunk overlap issues when writing with Dask.
+
+**Parameters:**
+- `dimension_size` (int): Size of the dimension to chunk
+- `target_chunk_size` (int): Desired chunk size
+
+**Returns:**
+- `int`: Aligned chunk size that divides evenly into dimension_size
+
+**Example:**
+```python
+from eopf_geozarr.conversion.utils import calculate_aligned_chunk_size
+
+# For a dimension of size 5490 with target chunk size 3660
+aligned_size = calculate_aligned_chunk_size(5490, 3660)  # Returns 2745
+```
 
 #### `is_grid_mapping_variable`
 

pyproject.toml

@@ -30,7 +30,8 @@ classifiers = [
 requires-python = ">=3.11"
 dependencies = [
     "pydantic-zarr@git+https://github.com/zarr-developers/pydantic-zarr",
-    "zarr>=3.0.10",
+    # "zarr>=3.0.10",
+    "zarr@git+https://github.com/zarr-developers/zarr-python",
     "xarray>=2025.7.1",
     "dask[array,distributed]>=2025.5.1",
     "numpy>=2.3.1",