Refactor S3 JSON metadata handling: replace direct S3 file read with a dedicated read_s3_json_metadata function for improved clarity and reusability.

Author: emmanuelmathot

eopf_geozarr/conversion/geozarr.py

@@ -695,12 +695,7 @@ def create_geozarr_compliant_multiscales(
     # Handle S3 vs local paths for JSON metadata
     if s3_utils.is_s3_path(output_path):
         # For S3, use s3fs with proper configuration
-        import s3fs
-        storage_options = s3_utils.get_s3_storage_options(zarr_json_path)
-        fs = s3fs.S3FileSystem(**storage_options)
-        
-        with fs.open(zarr_json_path, "r") as f:
-            zarr_json = json.load(f)
+        zarr_json = s3_utils.read_s3_json_metadata(zarr_json_path)
 
         zarr_json["attributes"]["multiscales"] = {
             "tile_matrix_set": tile_matrix_set,

eopf_geozarr/conversion/s3_utils.py

@@ -149,6 +149,44 @@ def write_s3_json_metadata(s3_path: str, metadata: Dict[str, Any], **s3_kwargs)
     with fs.open(s3_path, "w") as f:
         f.write(json_content)
 
+def read_s3_json_metadata(s3_path: str, **s3_kwargs) -> Dict[str, Any]:
+    """
+    Read JSON metadata from S3.
+
+    Parameters
+    ----------
+    s3_path : str
+        S3 path for the JSON file
+    **s3_kwargs
+        Additional keyword arguments for s3fs.S3FileSystem
+
+    Returns
+    -------
+    dict
+        Parsed JSON metadata
+    """
+    # Set up default S3 configuration
+    default_s3_kwargs = {
+        "anon": False,
+        "use_ssl": True,
+        "asynchronous": False,  # Force synchronous mode
+        "client_kwargs": {
+            "region_name": os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
+        }
+    }
+    
+    # Add custom endpoint support (e.g., for OVH Cloud)
+    if "AWS_S3_ENDPOINT" in os.environ:
+        default_s3_kwargs["endpoint_url"] = os.environ["AWS_S3_ENDPOINT"]
+        default_s3_kwargs["client_kwargs"]["endpoint_url"] = os.environ["AWS_S3_ENDPOINT"]
+    
+    s3_config = {**default_s3_kwargs, **s3_kwargs}
+    fs = s3fs.S3FileSystem(**s3_config)
+    
+    with fs.open(s3_path, "r") as f:
+        content = f.read()
+    
+    return json.loads(content)
 
 def s3_path_exists(s3_path: str, **s3_kwargs) -> bool:
     """