Refactor GeoZarr conversion utilities

- Updated launch configuration for GeoZarr conversion to use S2L1C instead of S2L2A.
- Moved S3 utility functions from `s3_utils.py` to `fs_utils.py` for better organization.
- Simplified path handling by introducing unified functions for storage options and path normalization.
- Removed redundant S3 checks in various functions, streamlining the code for local and S3 paths.
- Enhanced metadata handling with unified read/write functions for JSON metadata across different path types.

Author: emmanuelmathot

.vscode/launch.json

@@ -60,16 +60,16 @@
         },
         {
             // eopf_geozarr convert https://objectstore.eodc.eu:2222/e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr /tmp/tmp7mmjkjk3/s2b_subset_test.zarr --groups /measurements/reflectance/r10m --spatial-chunk 512 --min-dimension 128 --tile-width 256 --max-retries 2 --verbose
-            "name": "Convert to GeoZarr S2L2A (S3)",
+            "name": "Convert to GeoZarr S2L1C (S3)",
             "type": "debugpy",
             "request": "launch",
             "module": "eopf_geozarr",
             "args": [
                 "convert",
-                "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202507-s02msil2a/04/products/cpm_v256/S2A_MSIL2A_20250704T094051_N0511_R036_T33SWB_20250704T115824.zarr",
-                "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/s2b_test.zarr",
-                "--groups", "/measurements/reflectance/r10m", "/measurements/reflectance/r20m", "/measurements/reflectance/r60m", "/quality/l2a_quicklook/r10m",
-                "--spatial-chunk", "4096",
+                "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202507-s02msil1c/25/products/cpm_v256/S2A_MSIL1C_20250725T091041_N0511_R050_T34SGE_20250725T101000.zarr",
+                "s3://esa-zarr-sentinel-explorer-fra/tests-output/eopf_geozarr/S2A_MSIL1C_20250725T091041_N0511_R050_T34SGE_20250725T101000.zarr",
+                "--groups", "/measurements/reflectance/r10m", "/measurements/reflectance/r20m", "/measurements/reflectance/r60m", "/quality/l1c_quicklook/r10m",
+                "--spatial-chunk", "1024",
                 "--min-dimension", "256",
                 "--tile-width", "256",
                 "--max-retries", "2",

eopf_geozarr/cli.py

@@ -13,7 +13,7 @@
 import xarray as xr
 
 from . import create_geozarr_dataset
-from .conversion import is_s3_path, validate_s3_access, get_s3_credentials_info
+from .conversion.fs_utils import is_s3_path, validate_s3_access, get_s3_credentials_info
 
 
 def setup_dask_cluster(enable_dask: bool, verbose: bool = False) -> Optional[object]:

eopf_geozarr/conversion/__init__.py

@@ -8,7 +8,7 @@
     recursive_copy,
     setup_datatree_metadata_geozarr_spec_compliant,
 )
-from .s3_utils import (
+from .fs_utils import (
     create_s3_store,
     get_s3_credentials_info,
     is_s3_path,

eopf_geozarr/conversion/s3_utils.py eopf_geozarr/conversion/fs_utils.pyeopf_geozarr/conversion/s3_utils.py renamed to eopf_geozarr/conversion/fs_utils.py

@@ -125,6 +125,57 @@ def get_s3_storage_options(s3_path: str, **s3_kwargs) -> Dict[str, Any]:
     return s3_config
 
 
+def get_storage_options(path: str, **kwargs) -> Optional[Dict[str, Any]]:
+    """
+    Get storage options for any URL type, leveraging fsspec as the abstraction layer.
+    
+    This function eliminates the need for if/else branching by returning appropriate
+    storage options based on the URL protocol.
+
+    Parameters
+    ----------
+    path : str
+        Path or URL (local path, s3://, etc.)
+    **kwargs
+        Additional keyword arguments for the storage backend
+
+    Returns
+    -------
+    Optional[Dict[str, Any]]
+        Storage options dictionary for xarray/zarr, or None for local paths
+    """
+    if is_s3_path(path):
+        return get_s3_storage_options(path, **kwargs)
+    # For local paths, return None (no storage options needed)
+    # Future protocols (gcs://, azure://, etc.) can be added here
+    return None
+
+
+def normalize_path(path: str) -> str:
+    """
+    Normalize any path type (local or remote URL).
+    
+    This function handles path normalization for all filesystem types,
+    ensuring proper path formatting and removing issues like double slashes.
+    
+    Parameters
+    ----------
+    path : str
+        Path to normalize
+
+    Returns
+    -------
+    str
+        Normalized path
+    """
+    if is_s3_path(path):
+        return normalize_s3_path(path)
+    else:
+        # For local paths, normalize by removing double slashes and cleaning up
+        import os.path
+        return os.path.normpath(path)
+
+
 def create_s3_store(s3_path: str, **s3_kwargs) -> str:
     """
     Create an S3 path with storage options for Zarr operations.
@@ -346,3 +397,123 @@ def validate_s3_access(s3_path: str, **s3_kwargs) -> tuple[bool, Optional[str]]:
 
     except Exception as e:
         return False, str(e)
+
+
+def get_filesystem(path: str, **kwargs):
+    """
+    Get the appropriate fsspec filesystem for any path type.
+    
+    Parameters
+    ----------
+    path : str
+        Path or URL (local path, s3://, etc.)
+    **kwargs
+        Additional keyword arguments for the filesystem
+
+    Returns
+    -------
+    fsspec.AbstractFileSystem
+        Filesystem instance
+    """
+    import fsspec
+    
+    if is_s3_path(path):
+        # Get S3 storage options and use them for fsspec
+        storage_options = get_s3_storage_options(path, **kwargs)
+        return fsspec.filesystem('s3', **storage_options)
+    else:
+        # For local paths, use the local filesystem
+        return fsspec.filesystem('file')
+
+
+def write_json_metadata(path: str, metadata: Dict[str, Any], **kwargs) -> None:
+    """
+    Write JSON metadata to any path type using fsspec.
+    
+    Parameters
+    ----------
+    path : str
+        Path where to write the JSON file (local path or URL)
+    metadata : dict
+        Metadata dictionary to write as JSON
+    **kwargs
+        Additional keyword arguments for the filesystem
+    """
+    fs = get_filesystem(path, **kwargs)
+    
+    # Ensure parent directory exists for local paths
+    if not is_s3_path(path):
+        parent_dir = os.path.dirname(path)
+        if parent_dir:
+            fs.makedirs(parent_dir, exist_ok=True)
+    
+    # Write JSON content using fsspec
+    json_content = json.dumps(metadata, indent=2)
+    with fs.open(path, "w") as f:
+        f.write(json_content)
+
+
+def read_json_metadata(path: str, **kwargs) -> Dict[str, Any]:
+    """
+    Read JSON metadata from any path type using fsspec.
+    
+    Parameters
+    ----------
+    path : str
+        Path to the JSON file (local path or URL)
+    **kwargs
+        Additional keyword arguments for the filesystem
+
+    Returns
+    -------
+    dict
+        Parsed JSON metadata
+    """
+    fs = get_filesystem(path, **kwargs)
+    
+    with fs.open(path, "r") as f:
+        content = f.read()
+    
+    return json.loads(content)
+
+
+def path_exists(path: str, **kwargs) -> bool:
+    """
+    Check if a path exists using fsspec.
+    
+    Parameters
+    ----------
+    path : str
+        Path to check (local path or URL)
+    **kwargs
+        Additional keyword arguments for the filesystem
+
+    Returns
+    -------
+    bool
+        True if the path exists
+    """
+    fs = get_filesystem(path, **kwargs)
+    return fs.exists(path)
+
+
+def open_zarr_group(path: str, mode: str = "r", **kwargs) -> zarr.Group:
+    """
+    Open a Zarr group from any path type using unified storage options.
+    
+    Parameters
+    ----------
+    path : str
+        Path to the Zarr group (local path or URL)
+    mode : str, default "r"
+        Access mode ("r", "r+", "w", "a")
+    **kwargs
+        Additional keyword arguments for the storage backend
+
+    Returns
+    -------
+    zarr.Group
+        Zarr group
+    """
+    storage_options = get_storage_options(path, **kwargs)
+    return zarr.open_group(path, mode=mode, zarr_format=3, storage_options=storage_options)