Improve consistency and engine keyword argument for to_netcdf()

This PR introduces a bug fix and a breaking changes:

1. The default backend ``engine`` used by `Dataset.to_netcdf`
   and `DataTree.to_netcdf` is now chosen consistently with
   `open_dataset` and `open_datatree`, using whichever netCDF
   libraries are available and preferring netCDF4 to h5netcdf to scipy.
   Previously, `DataTree.to_netcdf` was hard-coded to use h5netcdf.
2. The return value of `Dataset.to_netcdf` without ``path`` is
   now a ``memoryview`` object instead of ``bytes``. This removes an unnecessary
   memory copy and ensures consistency when using either ``engine="scipy"`` or
   ``engine="h5netcdf"``.

Fixes #10654

Author: shoyer

doc/whats-new.rst

@@ -24,6 +24,19 @@ New Features
 Breaking changes
 ~~~~~~~~~~~~~~~~
 
+- The default backend ``engine`` used by :py:meth:`Dataset.to_netcdf`
+  and :py:meth:`DataTree.to_netcdf` is now chosen consistently with
+  :py:func:`open_dataset` and :py:func:`open_datatree`, using whichever netCDF
+  libraries are available and preferring netCDF4 to h5netcdf to scipy
+  (:issue:`10654`). Previously, :py:meth:`DataTree.to_netcdf` was hard-coded to
+  use h5netcdf.
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+- The return value of :py:meth:`Dataset.to_netcdf` without ``path`` is
+  now a ``memoryview`` object instead of ``bytes``. This removes an unnecessary
+  memory copy and ensures consistency when using either ``engine="scipy"`` or
+  ``engine="h5netcdf"``. If you need a bytes object, simply wrap the return
+  value of ``to_netcdf()`` with ``bytes()``.
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
 
 Deprecations
 ~~~~~~~~~~~~
@@ -41,6 +54,7 @@ Bug fixes
   (:issue:`10637`).
   By `Stephan Hoyer <https://github.com/shoyer>`_.
 
+
 Documentation
 ~~~~~~~~~~~~~
 

xarray/backends/api.py

@@ -17,7 +17,6 @@
 from typing import (
     TYPE_CHECKING,
     Any,
-    Final,
     Literal,
     TypeVar,
     Union,
@@ -98,69 +97,44 @@
 DATAARRAY_NAME = "__xarray_dataarray_name__"
 DATAARRAY_VARIABLE = "__xarray_dataarray_variable__"
 
-ENGINES = {
-    "netcdf4": backends.NetCDF4DataStore.open,
-    "scipy": backends.ScipyDataStore,
-    "pydap": backends.PydapDataStore.open,
-    "h5netcdf": backends.H5NetCDFStore.open,
-    "zarr": backends.ZarrStore.open_group,
-}
-
-
-def _get_default_engine_remote_uri() -> Literal["netcdf4", "pydap"]:
-    engine: Literal["netcdf4", "pydap"]
-    try:
-        import netCDF4  # noqa: F401
-
-        engine = "netcdf4"
-    except ImportError:  # pragma: no cover
-        try:
-            import pydap  # noqa: F401
 
-            engine = "pydap"
-        except ImportError as err:
-            raise ValueError(
-                "netCDF4 or pydap is required for accessing remote datasets via OPeNDAP"
-            ) from err
-    return engine
-
-
-def _get_default_engine_gz() -> Literal["scipy"]:
-    try:
-        import scipy  # noqa: F401
+def get_default_netcdf_write_engine(
+    format: T_NetcdfTypes | None,
+    to_file_or_memoryview: bool,
+) -> Literal["netcdf4", "h5netcdf", "scipy"]:
+    """Return the default netCDF library to use for writing a netCDF file."""
+    module_names = {
+        "netcdf4": "netCDF4",
+        "scipy": "scipy",
+        "h5netcdf": "h5netcdf",
+    }
 
-        engine: Final = "scipy"
-    except ImportError as err:  # pragma: no cover
-        raise ValueError("scipy is required for accessing .gz files") from err
-    return engine
+    candidates = list(plugins.NETCDF_BACKENDS_ORDER)
 
+    if format is not None:
+        if format.upper().startswith("NETCDF3"):
+            candidates.remove("h5netcdf")
+        elif format.upper().startswith("NETCDF4"):
+            candidates.remove("scipy")
+        else:
+            raise ValueError(f"unexpected {format=}")
 
-def _get_default_engine_netcdf() -> Literal["netcdf4", "h5netcdf", "scipy"]:
-    candidates: list[tuple[str, str]] = [
-        ("netcdf4", "netCDF4"),
-        ("h5netcdf", "h5netcdf"),
-        ("scipy", "scipy.io.netcdf"),
-    ]
+    if to_file_or_memoryview:
+        candidates.remove("netcdf4")
 
-    for engine, module_name in candidates:
+    for engine in candidates:
+        module_name = module_names[engine]
         if importlib.util.find_spec(module_name) is not None:
             return cast(Literal["netcdf4", "h5netcdf", "scipy"], engine)
 
+    format_str = f" with {format=}" if format is not None else ""
+    libraries = ", ".join(module_names[c] for c in candidates)
     raise ValueError(
-        "cannot read or write NetCDF files because none of "
-        "'netCDF4-python', 'h5netcdf', or 'scipy' are installed"
+        f"cannot write NetCDF files{format_str} because none of the suitable "
+        f"backend libraries ({libraries}) are installed"
     )
 
 
-def _get_default_engine(path: str, allow_remote: bool = False) -> T_NetcdfEngine:
-    if allow_remote and is_remote_uri(path):
-        return _get_default_engine_remote_uri()  # type: ignore[return-value]
-    elif path.endswith(".gz"):
-        return _get_default_engine_gz()
-    else:
-        return _get_default_engine_netcdf()
-
-
 def _validate_dataset_names(dataset: Dataset) -> None:
     """DataArray.name and Dataset keys must be a string or None"""
 
@@ -1958,7 +1932,7 @@ def to_netcdf(
     multifile: Literal[False] = False,
     invalid_netcdf: bool = False,
     auto_complex: bool | None = None,
-) -> bytes | memoryview: ...
+) -> memoryview: ...
 
 
 # compute=False returns dask.Delayed
@@ -2051,7 +2025,7 @@ def to_netcdf(
     multifile: bool = False,
     invalid_netcdf: bool = False,
     auto_complex: bool | None = None,
-) -> tuple[ArrayWriter, AbstractDataStore] | bytes | memoryview | Delayed | None: ...
+) -> tuple[ArrayWriter, AbstractDataStore] | memoryview | Delayed | None: ...
 
 
 def to_netcdf(
@@ -2067,41 +2041,22 @@ def to_netcdf(
     multifile: bool = False,
     invalid_netcdf: bool = False,
     auto_complex: bool | None = None,
-) -> tuple[ArrayWriter, AbstractDataStore] | bytes | memoryview | Delayed | None:
+) -> tuple[ArrayWriter, AbstractDataStore] | memoryview | Delayed | None:
     """This function creates an appropriate datastore for writing a dataset to
     disk as a netCDF file
 
     See `Dataset.to_netcdf` for full API docs.
 
     The ``multifile`` argument is only for the private use of save_mfdataset.
     """
-    if isinstance(path_or_file, os.PathLike):
-        path_or_file = os.fspath(path_or_file)
-
     if encoding is None:
         encoding = {}
 
-    if isinstance(path_or_file, str):
-        if engine is None:
-            engine = _get_default_engine(path_or_file)
-        path_or_file = _normalize_path(path_or_file)
-    else:
-        # writing to bytes/memoryview or a file-like object
-        if engine is None:
-            # TODO: only use 'scipy' if format is None or a netCDF3 format
-            engine = "scipy"
-        elif engine not in ("scipy", "h5netcdf"):
-            raise ValueError(
-                "invalid engine for creating bytes/memoryview or writing to a "
-                f"file-like object with to_netcdf: {engine!r}. Only "
-                "engine=None, engine='scipy' and engine='h5netcdf' is "
-                "supported."
-            )
-        if not compute:
-            raise NotImplementedError(
-                "to_netcdf() with compute=False is not yet implemented when "
-                "returning bytes"
-            )
+    path_or_file = _normalize_path(path_or_file)
+
+    if engine is None:
+        to_file_or_memoryview = not isinstance(path_or_file, str)
+        engine = get_default_netcdf_write_engine(format, to_file_or_memoryview)
 
     # validate Dataset keys, DataArray names, and attr keys/values
     _validate_dataset_names(dataset)
@@ -2121,6 +2076,11 @@ def to_netcdf(
         )
 
     if path_or_file is None:
+        if not compute:
+            raise NotImplementedError(
+                "to_netcdf() with compute=False is not yet implemented when "
+                "returning a memoryview"
+            )
         target = BytesIOProxy()
     else:
         target = path_or_file  # type: ignore[assignment]

xarray/backends/plugins.py

@@ -18,7 +18,7 @@
     from xarray.backends.common import AbstractDataStore
     from xarray.core.types import ReadBuffer
 
-STANDARD_BACKENDS_ORDER = ["netcdf4", "h5netcdf", "scipy"]
+NETCDF_BACKENDS_ORDER = ["netcdf4", "h5netcdf", "scipy"]
 
 
 def remove_duplicates(entrypoints: EntryPoints) -> list[EntryPoint]:
@@ -92,7 +92,7 @@ def sort_backends(
     backend_entrypoints: dict[str, type[BackendEntrypoint]],
 ) -> dict[str, type[BackendEntrypoint]]:
     ordered_backends_entrypoints = {}
-    for be_name in STANDARD_BACKENDS_ORDER:
+    for be_name in NETCDF_BACKENDS_ORDER:
         if backend_entrypoints.get(be_name) is not None:
             ordered_backends_entrypoints[be_name] = backend_entrypoints.pop(be_name)
     ordered_backends_entrypoints.update(

xarray/backends/scipy_.py

@@ -30,7 +30,6 @@
     Frozen,
     FrozenDict,
     close_on_error,
-    emit_user_level_warning,
     module_available,
     try_read_magic_number_from_file_or_path,
 )
@@ -169,20 +168,9 @@ def __init__(
         self.lock = ensure_lock(lock)
 
         if isinstance(filename_or_obj, BytesIOProxy):
-            emit_user_level_warning(
-                "return value of to_netcdf() without a target for "
-                "engine='scipy' is currently bytes, but will switch to "
-                "memoryview in a future version of Xarray. To silence this "
-                "warning, use the following pattern or switch to "
-                "to_netcdf(engine='h5netcdf'):\n"
-                "    target = io.BytesIO()\n"
-                "    dataset.to_netcdf(target)\n"
-                "    result = target.getbuffer()",
-                FutureWarning,
-            )
             source = filename_or_obj
             filename_or_obj = io.BytesIO()
-            source.getvalue = filename_or_obj.getvalue
+            source.getvalue = filename_or_obj.getbuffer
 
         if isinstance(filename_or_obj, str):  # path
             manager = CachingFileManager(

xarray/core/dataarray.py

@@ -4067,7 +4067,7 @@ def to_netcdf(
         compute: bool = True,
         invalid_netcdf: bool = False,
         auto_complex: bool | None = None,
-    ) -> bytes | memoryview: ...
+    ) -> memoryview: ...
 
     # compute=False returns dask.Delayed
     @overload
@@ -4131,17 +4131,15 @@ def to_netcdf(
         compute: bool = True,
         invalid_netcdf: bool = False,
         auto_complex: bool | None = None,
-    ) -> bytes | memoryview | Delayed | None:
+    ) -> memoryview | Delayed | None:
         """Write DataArray contents to a netCDF file.
 
         Parameters
         ----------
-        path : str, path-like or None, optional
-            Path to which to save this dataset. File-like objects are only
-            supported by the scipy engine. If no path is provided, this
-            function returns the resulting netCDF file as bytes; in this case,
-            we need to use scipy, which does not support netCDF version 4 (the
-            default format becomes NETCDF3_64BIT).
+        path : str, path-like, file-like or None, optional
+            Path to which to save this datatree, or a file-like object to write
+            it to (which must support read and write and be seekable) or None
+            (default) to return in-memory bytes as a memoryview.
         mode : {"w", "a"}, default: "w"
             Write ('w') or append ('a') mode. If mode='w', any existing file at
             this location will be overwritten. If mode='a', existing variables
@@ -4201,7 +4199,7 @@ def to_netcdf(
 
         Returns
         -------
-            * ``bytes`` or ``memoryview`` if path is None
+            * ``memoryview`` if path is None
             * ``dask.delayed.Delayed`` if compute is False
             * None otherwise
 

xarray/core/dataset.py

@@ -1953,7 +1953,7 @@ def to_netcdf(
         compute: bool = True,
         invalid_netcdf: bool = False,
         auto_complex: bool | None = None,
-    ) -> bytes | memoryview: ...
+    ) -> memoryview: ...
 
     # compute=False returns dask.Delayed
     @overload
@@ -2017,17 +2017,15 @@ def to_netcdf(
         compute: bool = True,
         invalid_netcdf: bool = False,
         auto_complex: bool | None = None,
-    ) -> bytes | memoryview | Delayed | None:
+    ) -> memoryview | Delayed | None:
         """Write dataset contents to a netCDF file.
 
         Parameters
         ----------
-        path : str, path-like or file-like, optional
-            Path to which to save this dataset. File-like objects are only
-            supported by the scipy engine. If no path is provided, this
-            function returns the resulting netCDF file as bytes; in this case,
-            we need to use scipy, which does not support netCDF version 4 (the
-            default format becomes NETCDF3_64BIT).
+        path : str, path-like, file-like or None, optional
+            Path to which to save this datatree, or a file-like object to write
+            it to (which must support read and write and be seekable) or None
+            (default) to return in-memory bytes as a memoryview.
         mode : {"w", "a"}, default: "w"
             Write ('w') or append ('a') mode. If mode='w', any existing file at
             this location will be overwritten. If mode='a', existing variables
@@ -2089,7 +2087,7 @@ def to_netcdf(
 
         Returns
         -------
-            * ``bytes`` or ``memoryview`` if path is None
+            * ``memoryview`` if path is None
             * ``dask.delayed.Delayed`` if compute is False
             * ``None`` otherwise
 

xarray/core/datatree_io.py

@@ -6,8 +6,10 @@
 from typing import TYPE_CHECKING, Any, Literal, get_args
 
 from xarray.backends.api import (
+    _normalize_path,
     delayed_close_after_writes,
     dump_to_store,
+    get_default_netcdf_write_engine,
     get_writable_netcdf_store,
     get_writable_zarr_store,
 )
@@ -50,8 +52,14 @@ def _datatree_to_netcdf(
             "DataTree.to_netcdf only supports the netcdf4 and h5netcdf engines"
         )
 
+    filepath = _normalize_path(filepath)
+
     if engine is None:
-        engine = "h5netcdf"
+        to_file_or_memoryview = not isinstance(filepath, str)
+        engine = get_default_netcdf_write_engine(
+            format="NETCDF4",  # required for supporting groups
+            to_file_or_memoryview=to_file_or_memoryview,
+        )  # type: ignore[assignment]
 
     if group is not None:
         raise NotImplementedError(
@@ -70,6 +78,11 @@ def _datatree_to_netcdf(
         )
 
     if filepath is None:
+        if not compute:
+            raise NotImplementedError(
+                "to_netcdf() with compute=False is not yet implemented when "
+                "returning bytes"
+            )
         # No need to use BytesIOProxy here because the legacy scipy backend
         # cannot write netCDF files with groups
         target = io.BytesIO()