Merge pull request #341 from boutproject/ci-apt-update

Fix CI: apt update on runner, file lock race condition

Author: bendudson

.github/workflows/master.yml

@@ -32,6 +32,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        sudo apt-get update
         sudo apt-get -y install libhdf5-dev
     - name: Install package
       run: |
@@ -61,6 +62,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        sudo apt-get update
         sudo apt-get -y install libhdf5-dev
         pip install xarray~=2023.12.0 pandas~=2.2.0
     - name: Install package

.github/workflows/pythonpackage.yml

@@ -32,14 +32,15 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        sudo apt-get update
         sudo apt-get -y install libhdf5-dev
     - name: Install package
       run: |
         pip install -e .[calc,tests] --no-binary h5netcdf --no-binary h5py
         pip install git+https://github.com/dschwoerer/xarray@netcdf4-locking-closing
     - name: Test with pytest
       run: |
-        pytest -vv --long --cov
+        timeout -s INT 90m pytest -vvv --long --cov --full-trace
 
 
   # test with oldest supported version of packages
@@ -61,6 +62,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        sudo apt-get update
         sudo apt-get -y install libhdf5-dev
         pip install xarray~=2023.12.0 pandas~=2.2.0
     - name: Install package

xbout/boutdataset.py

@@ -29,6 +29,7 @@
 from .region import _from_region
 from .utils import (
     _add_cartesian_coordinates,
+    _check_filetype,
     _get_bounding_surfaces,
     _split_into_restarts,
 )
@@ -719,8 +720,7 @@ def remove_yboundaries(self, **kwargs):
                 new_metadata = variables[-1].metadata
             elif ycoord in self.data[v].dims:
                 raise ValueError(
-                    f"{v} only has a {ycoord}-dimension so cannot split "
-                    f"into regions."
+                    f"{v} only has a {ycoord}-dimension so cannot split into regions."
                 )
             else:
                 variable = self.data[v]
@@ -788,6 +788,9 @@ def save(
         ----------
         savepath : str, optional
         filetype : str, optional
+            netCDF format passed to xarray. NOT THE SAME as "engine", which
+            defaults to h5netcdf via ``_check_filetype()``.
+            See https://docs.xarray.dev/en/latest/generated/xarray.Dataset.to_netcdf.html
         variables : list of str, optional
             Variables from the dataset to save. Default is to save all of them.
         separate_vars: bool, optional
@@ -910,6 +913,7 @@ def dict_to_attrs(obj, section):
                     single_var_ds.to_netcdf(
                         path=str(var_savepath),
                         format=filetype,
+                        engine=_check_filetype(Path(var_savepath)),
                         compute=True,
                         encoding=var_encoding,
                     )
@@ -923,7 +927,11 @@ def dict_to_attrs(obj, section):
             print("Saving data...")
             with ProgressBar():
                 to_save.to_netcdf(
-                    path=savepath, format=filetype, compute=True, encoding=encoding
+                    path=savepath,
+                    engine=_check_filetype(Path(savepath)),
+                    format=filetype,
+                    compute=True,
+                    encoding=encoding,
                 )
 
         return
@@ -992,7 +1000,12 @@ def to_restart(
         )
 
         with ProgressBar():
-            xr.save_mfdataset(restart_datasets, paths, compute=True)
+            xr.save_mfdataset(
+                restart_datasets,
+                paths,
+                compute=True,
+                engine=_check_filetype(paths[0]),
+            )
 
     def animate_list(
         self,

xbout/load.py

@@ -197,12 +197,15 @@ def open_boutdataset(
                 # xr.open_mfdataset only accepts glob patterns as
                 # strings, not Path objects
                 datapath = str(datapath)
+            _, filetype = _expand_filepaths(datapath)
+            reload_kwargs = dict(kwargs)
+            reload_kwargs.setdefault("engine", file_engine or filetype)
             ds = xr.open_mfdataset(
                 datapath,
                 chunks=chunks,
                 combine="by_coords",
                 data_vars="minimal",
-                **kwargs,
+                **reload_kwargs,
             )
         elif input_type == "reload_fake":
             ds = xr.combine_by_coords(datapath, data_vars="minimal").chunk(chunks)