Refactor Zarr store writing

.github/workflows/ci.yml

@@ -83,13 +83,14 @@ jobs:
 
     - name: Install dependencies
       run: |
+       sudo apt-get update
+        sudo apt-get install -y proj-bin gdal-bin libgdal-dev
         python -m pip install --upgrade pip
-        pip install -e ".[dev,test]"
+        pip install -e ".[dev,test]" --only-binary=:all:
 
     - name: Run network tests
       run: |
         python -m pytest eopf_geozarr/tests/ -v --tb=short -m "network"
-
   security:
     runs-on: ubuntu-latest
     steps:

.github/workflows/docs.yml

@@ -19,8 +19,10 @@ jobs:
 
     - name: Install dependencies
       run: |
+        sudo apt-get update
+        sudo apt-get install -y proj-bin gdal-bin libgdal-dev
         python -m pip install --upgrade pip
-        pip install -e ".[docs]"
+        pip install -e ".[docs]" --only-binary=:all:
 
     - name: Build documentation
       run: |

.pre-commit-config.yaml

@@ -1,44 +1,29 @@
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: v0.24.1
     hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-yaml
-      - id: check-added-large-files
-      - id: check-merge-conflict
-      - id: check-toml
-      - id: debug-statements
+      - id: validate-pyproject
 
-  - repo: https://github.com/psf/black
-    rev: 23.7.0
-    hooks:
-      - id: black
-        language_version: python3
-
-  - repo: https://github.com/pycqa/isort
+  - repo: https://github.com/PyCQA/isort
     rev: 5.12.0
     hooks:
       - id: isort
-        args: ["--profile", "black"]
+        language_version: python
 
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.4
     hooks:
-      - id: flake8
-        additional_dependencies: [flake8-docstrings]
-        args: [--max-line-length=150, --extend-ignore=E203]
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.11.2
     hooks:
       - id: mypy
-        additional_dependencies: [types-requests, types-setuptools]
-        args: [--ignore-missing-imports]
-
-  - repo: https://github.com/pycqa/bandit
-    rev: 1.7.5
-    hooks:
-      - id: bandit
-        args: ["-c", "pyproject.toml"]
-        additional_dependencies: ["bandit[toml]"]
+        language_version: python
+        exclude: tests/.*
+        additional_dependencies:
+        - types-simplejson
+        - types-attrs
+        - pydantic~=2.0

.vscode/launch.json

@@ -55,7 +55,7 @@
                 "PYTHONPATH": "${workspaceFolder}/.venv/bin",
                 "AWS_PROFILE": "eopf-explorer",
                 "AWS_DEFAULT_REGION": "de",
-                "AWS_S3_ENDPOINT": "https://s3.de.io.cloud.ovh.net/"
+                "AWS_ENDPOINT_URL": "https://s3.de.io.cloud.ovh.net/"
             },
 
         },
@@ -84,7 +84,7 @@
                 "PYTHONPATH": "${workspaceFolder}/.venv/bin",
                 "AWS_PROFILE": "eopf-explorer",
                 "AWS_DEFAULT_REGION": "de",
-                "AWS_S3_ENDPOINT": "https://s3.de.io.cloud.ovh.net/"
+                "AWS_ENDPOINT_URL": "https://s3.de.io.cloud.ovh.net/"
             },
 
         },
@@ -96,10 +96,17 @@
             "module": "eopf_geozarr",
             "args": [
                 "convert",
-                "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202507-s02msil2a/04/products/cpm_v256/S2A_MSIL2A_20250704T094051_N0511_R036_T33SWB_20250704T115824.zarr",
-                "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/S2A_MSIL2A_20250704T094051_N0511_R036_T33SWB_20250704T115824.zarr",
+                // "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202507-s02msil2a/04/products/cpm_v256/S2A_MSIL2A_20250704T094051_N0511_R036_T33SWB_20250704T115824.zarr",
+                // "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202508-s02msil2a/04/products/cpm_v256/S2B_MSIL2A_20250804T103629_N0511_R008_T31TDH_20250804T130722.zarr",
+                // "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202508-s02msil2a/07/products/cpm_v256/S2B_MSIL2A_20250807T104619_N0511_R051_T31TDH_20250807T131144.zarr",
+                "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202508-s02msil2a/11/products/cpm_v256/S2C_MSIL2A_20250811T112131_N0511_R037_T29TPF_20250811T152216.zarr",
+                // "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/S2A_MSIL2A_20250704T094051_N0511_R036_T33SWB_20250704T115824.zarr",
+                // "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/S2B_MSIL2A_20250804T103629_N0511_R008_T31TDH_20250804T130722.zarr",
+                // "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/S2B_MSIL2A_20250807T104619_N0511_R051_T31TDH_20250807T131144.zarr",
+                "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/S2C_MSIL2A_20250811T112131_N0511_R037_T29TPF_20250811T152216.zarr",
                 "--groups", "/measurements/reflectance/r10m", "/measurements/reflectance/r20m", "/measurements/reflectance/r60m", "/quality/l2a_quicklook/r10m",
-                "--spatial-chunk", "1024",
+                "--crs-groups", "/conditions/geometry",
+                "--spatial-chunk", "512",
                 "--min-dimension", "256",
                 "--tile-width", "256",
                 "--max-retries", "2",
@@ -113,7 +120,7 @@
                 "PYTHONPATH": "${workspaceFolder}/.venv/bin",
                 "AWS_PROFILE": "eopf-explorer",
                 "AWS_DEFAULT_REGION": "de",
-                "AWS_S3_ENDPOINT": "https://s3.de.io.cloud.ovh.net/"
+                "AWS_ENDPOINT_URL": "https://s3.de.io.cloud.ovh.net/"
             },
 
         },
@@ -154,7 +161,7 @@
                 "PYTHONPATH": "${workspaceFolder}/.venv/bin",
                 "AWS_PROFILE": "eopf-explorer",
                 "AWS_DEFAULT_REGION": "de",
-                "AWS_S3_ENDPOINT": "https://s3.de.io.cloud.ovh.net/"
+                "AWS_ENDPOINT_URL": "https://s3.de.io.cloud.ovh.net/"
             },
 
         }

README.md

@@ -94,21 +94,21 @@ export AWS_DEFAULT_REGION=us-east-1
 export AWS_ACCESS_KEY_ID=your_ovh_access_key
 export AWS_SECRET_ACCESS_KEY=your_ovh_secret_key
 export AWS_DEFAULT_REGION=gra  # or other OVH region
-export AWS_S3_ENDPOINT=https://s3.gra.cloud.ovh.net  # OVH endpoint
+export AWS_ENDPOINT_URL=https://s3.gra.cloud.ovh.net  # OVH endpoint
 ```
 
 **For other S3-compatible providers:**
 ```bash
 export AWS_ACCESS_KEY_ID=your_access_key
 export AWS_SECRET_ACCESS_KEY=your_secret_key
 export AWS_DEFAULT_REGION=your_region
-export AWS_S3_ENDPOINT=https://your-s3-endpoint.com
+export AWS_ENDPOINT_URL=https://your-s3-endpoint.com
 ```
 
 **Alternative: AWS CLI Configuration**
 ```bash
 aws configure
-# Note: For custom endpoints, you'll still need to set AWS_S3_ENDPOINT
+# Note: For custom endpoints, you'll still need to set AWS_ENDPOINT_URL
 ```
 
 #### S3 Features
@@ -166,7 +166,7 @@ from eopf_geozarr import create_geozarr_dataset
 os.environ['AWS_ACCESS_KEY_ID'] = 'your_ovh_access_key'
 os.environ['AWS_SECRET_ACCESS_KEY'] = 'your_ovh_secret_key'
 os.environ['AWS_DEFAULT_REGION'] = 'gra'
-os.environ['AWS_S3_ENDPOINT'] = 'https://s3.gra.cloud.ovh.net'
+os.environ['AWS_ENDPOINT_URL'] = 'https://s3.gra.cloud.ovh.net'
 
 # Load your EOPF DataTree
 dt = xr.open_datatree("path/to/eopf/dataset.zarr", engine="zarr")

eopf_geozarr/__init__.py

@@ -9,7 +9,7 @@
     create_geozarr_dataset,
     downsample_2d_array,
     is_grid_mapping_variable,
-    recursive_copy,
+    iterative_copy,
     setup_datatree_metadata_geozarr_spec_compliant,
     validate_existing_band_data,
 )
@@ -20,7 +20,7 @@
     "__version__",
     "create_geozarr_dataset",
     "setup_datatree_metadata_geozarr_spec_compliant",
-    "recursive_copy",
+    "iterative_copy",
     "consolidate_metadata",
     "async_consolidate_metadata",
     "downsample_2d_array",