Update GeoZarr conversion: adjust spatial chunk and min dimension parameters, enhance S3 path handling with storage options, and improve JSON metadata writing for S3.

Author: emmanuelmathot

.vscode/launch.json

@@ -16,8 +16,8 @@
                 "https://objectstore.eodc.eu:2222/e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr",
                 "./tests-output/eopf_geozarr/s2b_test.zarr",
                 "--groups", "/measurements/reflectance/r10m", "/measurements/reflectance/r20m", "/measurements/reflectance/r60m", "/quality/l1c_quicklook/r10m",
-                "--spatial-chunk", "512",
-                "--min-dimension", "128",
+                "--spatial-chunk", "4096",
+                "--min-dimension", "256",
                 "--tile-width", "256",
                 "--max-retries", "2",
                 "--verbose"
@@ -51,8 +51,7 @@
             "console": "integratedTerminal",
             "env": {
                 "PYTHONPATH": "${workspaceFolder}/.venv/bin",
-                "AWS_ACCESS_KEY_ID": "secret",
-                "AWS_SECRET_ACCESS_KEY": "secret",
+                "AWS_PROFILE": "eopf-explorer",
                 "AWS_DEFAULT_REGION": "gra",
                 "AWS_S3_ENDPOINT": "https://s3.gra.io.cloud.ovh.net/" 
             },

eopf_geozarr/conversion/geozarr.py

@@ -526,7 +526,8 @@ def write_geozarr_group(
         if s3_utils.is_s3_path(output_path):
             if s3_utils.s3_path_exists(native_dataset_path):
                 store = s3_utils.create_s3_store(native_dataset_path)
-                existing_native_dataset = xr.open_zarr(store, zarr_format=3)
+                storage_options = s3_utils.get_s3_storage_options(native_dataset_path)
+                existing_native_dataset = xr.open_zarr(store, zarr_format=3, storage_options=storage_options)
                 print(f"Found existing native dataset at {native_dataset_path}")
         else:
             if os.path.exists(native_dataset_path):
@@ -696,8 +697,7 @@ def create_geozarr_compliant_multiscales(
             "tile_matrix_limits": tile_matrix_limits,
         }
 
-        with fs.open(zarr_json_path, "w") as f:
-            json.dump(zarr_json, f, indent=2)
+        s3_utils.write_s3_json_metadata(zarr_json_path, zarr_json)
     else:
         # Local file operations
         with open(zarr_json_path, "r") as f: