update dmrpp migration - soft imports and code refactor

Author: Mikejmnez

pyproject.toml

@@ -51,7 +51,9 @@ hdf = [
 ]
 
 # dmrpp
-dmrpp = ["pydap>=3.5.10"]
+dmrpp = [
+    "pydap @ git+https://github.com/pydap/pydap.git@refs/pull/697/head",
+]
 
 zarr = ["arro3-core"]
 

virtualizarr/parsers/dmrpp.py

@@ -1,23 +1,15 @@
-import base64
 import io
-import warnings
-from pathlib import Path
 from typing import Iterable
 from xml.etree import ElementTree as ET
 
-from obspec_utils.protocols import ReadableStore
+# from obspec_utils.protocols import ReadableStore
 from obspec_utils.readers import EagerStoreReader
 from obspec_utils.registry import ObjectStoreRegistry
-from pydap.parsers.dmr import DMRPPParser as _DMRPPParser
 
-from virtualizarr.manifests import (
-    ChunkManifest,
-    ManifestArray,
-    ManifestGroup,
-    ManifestStore,
-)
-from virtualizarr.manifests.utils import create_v3_array_metadata
-from virtualizarr.parsers.utils import encode_cf_fill_value
+from virtualizarr.manifests import ManifestStore
+from virtualizarr.utils import soft_import
+
+pydap = soft_import("pydap", "parsing dmrpp references", strict=False)
 
 
 class DMRPPParser:
@@ -71,159 +63,12 @@ def __call__(
             else url.removesuffix(".dmrpp")
         )
 
+        from pydap.virtualizarr.parser import DMRParser
+
         parser = DMRParser(
             root=ET.parse(stream).getroot(),
             data_filepath=url,
             skip_variables=self.skip_variables,
         )
         manifest_store = parser.parse_dataset(object_store=store, group=self.group)
         return manifest_store
-
-
-class DMRParser:
-    """
-    Parser for the OPeNDAP DMR++ XML format.
-    Reads groups, dimensions, coordinates, data variables, encoding, chunk manifests, and attributes.
-    Highly modular to allow support for older dmrpp schema versions. Includes many utility functions to extract
-    different information such as finding all variable tags, splitting hdf5 groups, parsing dimensions, and more.
-
-    OPeNDAP DMR++ homepage: https://docs.opendap.org/index.php/DMR%2B%2B
-    """
-
-    # DAP and DMRPP XML namespaces
-    _NS = {
-        "dap": "http://xml.opendap.org/ns/DAP/4.0#",
-        "dmrpp": "http://xml.opendap.org/dap/dmrpp/1.0.0#",
-    }
-
-    root: ET.Element
-    data_filepath: str
-
-    def __init__(
-        self,
-        root: ET.Element,
-        data_filepath: str | None = None,
-        skip_variables: Iterable[str] | None = None,
-    ):
-        """
-        Initialize the DMRParser with the given DMR++ file contents and source data file path.
-
-        Parameters
-        ----------
-        root
-            Root of the xml tree structure of a DMR++ file.
-        data_filepath
-            The path to the actual data file that will be set in the chunk manifests.
-            If None, the data file path is taken from the DMR++ file.
-        """
-        self.root = root
-        self.data_filepath = (
-            data_filepath if data_filepath is not None else self.root.attrib["name"]
-        )
-        self.skip_variables = skip_variables or ()
-        self._validation_issues: list[str] = []
-
-    def dmrparser(self) -> _DMRPPParser:
-        """Exposes the _DMRParser to external use (avoids breaking changes)"""
-        parser = _DMRPPParser(
-            root=self.root,
-            data_filepath=self.data_filepath,
-            skip_variables=self.skip_variables,
-        )
-        self._validation_issues = parser._validation_issues
-        return parser
-
-    def parse_dataset(
-        self,
-        object_store: ReadableStore,
-        group: str | None = None,
-    ) -> ManifestStore:
-        """
-        Parses the given file and creates a ManifestStore.
-
-        Parameters
-        ----------
-        group
-            The group to parse. Ignored if no groups are present, and the entire
-            dataset is parsed. If `None` or "/", and groups are present, the first group
-            is parsed.  If not `None` or "/", and no groups are present, a UserWarning
-            is issued indicating that the group will be ignored.
-
-        Returns
-        -------
-        ManifestStore
-
-        Examples
-        --------
-        Open a sample DMR++ file and parse the dataset
-        """
-        group = group or "/"
-        ngroups = len(self.root.findall("dap:Group", self._NS))
-
-        if ngroups == 0 and group != "/":
-            warnings.warn(
-                f"No groups in DMR++ file {self.data_filepath!r}; "
-                f"ignoring group parameter {group!r}"
-            )
-
-        group_path = Path("/") if ngroups == 0 else Path("/") / group.removeprefix("/")
-
-        dataset_element = self.dmrparser()._split_groups(self.root).get(group_path)
-
-        if dataset_element is None:
-            raise ValueError(
-                f"Group {group_path} not found in DMR++ file {self.data_filepath!r}"
-            )
-
-        # get two dictionary containing relevant metadata
-        vars_dict, attrs = self.dmrparser()._parse_dataset(dataset_element)
-
-        manifest_dict: dict[str, ManifestArray] = {}
-
-        for var in vars_dict.keys():
-            chunkmanifest = vars_dict[var].pop("chunkmanifest", None)
-            # remove opendap-related metadata
-            meta = dict(
-                [
-                    (k, v)
-                    for k, v in vars_dict[var].items()
-                    if k not in ["Maps", "fqn_dims"]
-                ]
-            )
-            if "_FillValue" in meta["attributes"]:
-                encoded_cf_fill_value = encode_cf_fill_value(
-                    meta["attributes"]["_FillValue"], meta["data_type"]
-                )
-                meta["attributes"]["_FillValue"] = encoded_cf_fill_value
-
-            if "inline" in meta:
-                # extract data already decoded into array/string
-                data = meta.pop("inline", None)
-                bdata = base64.b64encode(data)
-                # chunk_entry = ChunkEntry(
-                #     path="", offset=0, length=len(bdata), data=bdata
-                # )
-                # chunkmanifest = ChunkManifest(entries=chunk_entry)
-
-                chunks = {
-                    "0.0": {
-                        "path": "__inline__",
-                        "offset": 0,
-                        "length": len(bdata),
-                        "data": bdata,
-                    },
-                }
-                chunkmanifest = ChunkManifest(entries=chunks)
-            else:
-                chunkmanifest = ChunkManifest(chunkmanifest)
-
-            metadata = create_v3_array_metadata(**meta)
-            manifest_dict[var] = ManifestArray(
-                metadata=metadata, chunkmanifest=chunkmanifest
-            )
-
-        manifest_group = ManifestGroup(arrays=manifest_dict, attributes=attrs)
-        registry: ObjectStoreRegistry = ObjectStoreRegistry()
-        registry.register(self.data_filepath, object_store)
-
-        return ManifestStore(registry=registry, group=manifest_group)

virtualizarr/tests/__init__.py

@@ -44,3 +44,5 @@ def _importorskip(
 has_arro3, requires_arro3 = _importorskip("arro3.core")
 # The GribberishParser is new in gribberish 1.0.0.
 has_grib, requires_grib = _importorskip("gribberish", minversion="1.0.0")
+# The DMRPPParser is new in pydap 3.5.10
+has_pydap, requires_pydap = _importorskip("pydap")