Refactor S3 handling in GeoZarr conversion: normalize group paths to fix double slash issue, update spatial chunk and min dimension parameters, and enhance S3 group opening with storage options.

Author: emmanuelmathot

.vscode/launch.json

@@ -40,8 +40,8 @@
                 "https://objectstore.eodc.eu:2222/e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr",
                 "s3://esa-zarr-sentinel-explorer/tests-output/eopf_geozarr/s2b_test.zarr",
                 "--groups", "/measurements/reflectance/r10m", "/measurements/reflectance/r20m", "/measurements/reflectance/r60m", "/quality/l1c_quicklook/r10m",
-                "--spatial-chunk", "512",
-                "--min-dimension", "128",
+                "--spatial-chunk", "4096",
+                "--min-dimension", "256",
                 "--tile-width", "256",
                 "--max-retries", "2",
                 "--verbose"
@@ -51,8 +51,8 @@
             "console": "integratedTerminal",
             "env": {
                 "PYTHONPATH": "${workspaceFolder}/eopf_geozarr/.venv/bin",
-                "AWS_ACCESS_KEY_ID": "04fd31bb339247658e971fad3a1d879e",
-                "AWS_SECRET_ACCESS_KEY": "eb1b07f69b1d494abb947aa0bba8587e",
+                "AWS_ACCESS_KEY_ID": "secret",
+                "AWS_SECRET_ACCESS_KEY": "secret",
                 "AWS_DEFAULT_REGION": "gra",
                 "AWS_S3_ENDPOINT": "https://s3.gra.io.cloud.ovh.net/" 
             },

eopf_geozarr/conversion/geozarr.py

@@ -280,7 +280,11 @@ def recursive_copy(
         is_dataset = True
 
         # Handle S3 vs local paths for zarr operations
-        group_path = f"{output_path}/{group_prefix}"
+        # Fix double slash issue by normalizing the path
+        if group_prefix.startswith("/"):
+            group_path = f"{output_path}{group_prefix}"
+        else:
+            group_path = f"{output_path}/{group_prefix}"
         if s3_utils.is_s3_path(output_path):
             # For S3, use storage_options
             storage_options = s3_utils.get_s3_storage_options(group_path)
@@ -468,13 +472,14 @@ def write_geozarr_group(
 
     # Handle S3 vs local paths for zarr operations
     if s3_utils.is_s3_path(output_path):
-        # For S3, use the S3 store
-        store = s3_utils.create_s3_store(group_path)
+        # For S3, use storage_options
+        storage_options = s3_utils.get_s3_storage_options(group_path)
         dt.to_zarr(
-            store,
+            group_path,
             mode="a",  # Append mode to add to the group
             consolidated=False,  # No consolidate metadata
             zarr_format=3,  # Use Zarr format 3
+            storage_options=storage_options,
         )
     else:
         dt.to_zarr(
@@ -676,9 +681,10 @@ def create_geozarr_compliant_multiscales(
 
     # Handle S3 vs local paths for JSON metadata
     if s3_utils.is_s3_path(output_path):
-        # For S3, use s3fs to read/write JSON
+        # For S3, use s3fs with proper configuration
         import s3fs
-        fs = s3fs.S3FileSystem(anon=False)
+        storage_options = s3_utils.get_s3_storage_options(zarr_json_path)
+        fs = s3fs.S3FileSystem(**storage_options)
 
         with fs.open(zarr_json_path, "r") as f:
             zarr_json = json.load(f)

eopf_geozarr/conversion/s3_utils.py

@@ -188,7 +188,7 @@ def s3_path_exists(s3_path: str, **s3_kwargs) -> bool:
 
 def open_s3_zarr_group(s3_path: str, mode: str = "r", **s3_kwargs) -> zarr.Group:
     """
-    Open a Zarr group from S3.
+    Open a Zarr group from S3 using storage_options.
 
     Parameters
     ----------
@@ -204,8 +204,8 @@ def open_s3_zarr_group(s3_path: str, mode: str = "r", **s3_kwargs) -> zarr.Group
     zarr.Group
         Zarr group
     """
-    store = create_s3_store(s3_path, **s3_kwargs)
-    return zarr.open_group(store, mode=mode, zarr_format=3)
+    storage_options = get_s3_storage_options(s3_path, **s3_kwargs)
+    return zarr.open_group(s3_path, mode=mode, zarr_format=3, storage_options=storage_options)
 
 
 def get_s3_credentials_info() -> Dict[str, Optional[str]]: