diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 00000000..ba9c31f4 --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,113 @@ +name: Publish Python 🐍 distribution 📦 to PyPI + +on: push + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install pypa/build + run: python3 -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pyglider + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Sign the Python 🐍 distribution 📦 with Sigstore + and upload them to GitHub Release + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v2.1.1 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' + + publish-to-testpypi: + name: Publish Python 🐍 distribution 📦 to TestPyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/pyglider + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 920bf4c2..f9641edf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,42 +7,33 @@ jobs: strategy: matrix: os: ["ubuntu-latest"] - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 - - name: Cache conda - uses: actions/cache@v2 - env: - # Increase this value to reset cache if etc/example-environment.yml has not changed - CACHE_NUMBER: 0 + - uses: actions/checkout@v4 + - name: mamba setup enviroment + uses: mamba-org/setup-micromamba@v1.9.0 with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{hashFiles('environment.yml') }} - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pyglider-test + environment-name: test-env environment-file: tests/environment.yml - python-version: ${{ matrix.python-version }} - channel-priority: strict - use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + create-args: >- + python=${{ matrix.python-version }} - name: Conda info - shell: bash -l {0} + shell: micromamba-shell {0} run: conda info; conda list - name: install pyglider source - shell: bash -l {0} + shell: micromamba-shell {0} run: which pip; pip install -e . - name: Process seaexplorer - shell: bash -l {0} + shell: micromamba-shell {0} run: which python; cd tests/example-data/example-seaexplorer; make clean-all; python process_deploymentRealTime.py - name: Process slocum - shell: bash -l {0} + shell: micromamba-shell {0} run: which python; cd tests/example-data/example-slocum; make clean-all; python process_deploymentRealTime.py - name: Process seaexplorer-legato-flntu-arod-ad2cp - shell: bash -l {0} + shell: micromamba-shell {0} run: which python; cd tests/example-data/example-seaexplorer-legato-flntu-arod-ad2cp; make clean-all; python process_deploymentRealTime.py - name: Run tests - shell: bash -l {0} + shell: micromamba-shell {0} run: which python; pytest --cov --cov-report xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 \ No newline at end of file + uses: codecov/codecov-action@v4 diff --git a/.gitignore b/.gitignore index 0338b019..bc7ed6af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,26 @@ *.egg-info __pycache__ _build +build docs/pyglider dist *.png -.DS_Store \ No newline at end of file +.DS_Store + +# Ignore test items +tests/data/l0-profiles/ +tests/data/realtime_rawnc/ +tests/example-data/example-seaexplorer-raw/L0-timeseries-test-interp/ +tests/example-data/example-seaexplorer-raw/L0-timeseries-test/ +tests/example-data/example-seaexplorer-raw/delayed_rawnc/ +tests/example-data/example-seaexplorer-raw/deploymentDelayedInterp.yml +tests/example-data/example-seaexplorer/L0-timeseries-test-interp/ +tests/example-data/example-seaexplorer/L0-timeseries-test/ +tests/example-data/example-seaexplorer/bad_timebase.yml +tests/example-data/example-seaexplorer/deploymentRealtimeInterp.yml +tests/example-data/example-seaexplorer/no_timebase.yml +tests/example-data/example-seaexplorer/realtime_rawnc/ +tests/example-data/example-slocum/L0-profiles/ +tests/example-data/example-slocum/L0-timeseries/ +tests/example-data/report.json diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dfb1063e..a6835ca9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,12 +1,42 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required version: 2 -python: - version: 3.8 - install: - - method: pip - path: . - extra_requirements: - - docs - -# Build documentation in the docs/ directory with Sphinx + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx sphinx: - configuration: docs/conf.py \ No newline at end of file + configuration: docs/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +python: + install: + - requirements: docs-requirements.txt + # Install our python package before building the docs + - method: pip + path: . + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/getting-started-slocum.md b/docs/getting-started-slocum.md index 14f34071..e7dfaa66 100644 --- a/docs/getting-started-slocum.md +++ b/docs/getting-started-slocum.md @@ -2,19 +2,12 @@ ## Gather data -Slocum gliders have 4 types of files. For telemetry data there is `*.sbd` files for sensor data, and `*.tbd` for the glider's attitude and position data. These are called `*.dbd` and `*.ebd` respectively, when retrieved from the gliders' payload post deployment. These need to be made available in a single directory for `pyglider` to process. +Slocum gliders have 4 types of files. For telemetry data there are `*.tbd` files for sensor data, and `*.sbd` for the glider's attitude and position data. These are called `*.ebd` and `*.tbd` respectively, when retrieved from the gliders' payload post deployment. Modern gliders have compressed version of these, eg `*.tcd`, `*.scd` that *pyglider* should be able to parse. These data files need to be made available in a _single_ directory for *pyglider* to process. Note that on the glider they are often separated into `science/logs` and `flight/logs`. -Slocums also have a sensor cache file `*.cac`, all of which have randomized names. These are needed by the processing, and are usually stored in a separate cache directory. +Slocum gliders also have a sensor cache file `*.cac`, all of which have randomized names. These are needed by the processing, and are usually stored in a separate cache directory. -You can download and expand example data using `.get_example_data`: +You can download example data at which will add a local directory `example-data` to your current directory. -```python -import pyglider.example_data as pexamp - -pexamp.get_example_data('./') -``` - -which will add a local directory `example-data` to your current directory. ## Make a deployment configuration file @@ -33,8 +26,9 @@ The example script is relatively straight forward if there is no intermediate pr Data comes from an input directory, and is translated into a single CF-compliant netCDF timeseries file using the package [dbdreader](https://dbdreader.readthedocs.io/en/latest/). Finally individual profiles are saved and a 2-D 1-m grid in time-depth is saved. -```{note} There is a version that does not require `dbdreader` to do the initial conversion from the Dinkum format to netCDF. However it is quite slow, particularly for full-resolution datasets, and less robust. We suggest using the `slocum.raw_to_timeseries`. -``` +:::{note} +There is a version that does not require `dbdreader` to do the initial conversion from the Dinkum format to netCDF. However it is quite slow, particularly for full-resolution datasets, and less robust. We suggest using the `slocum.raw_to_timeseries`. +::: It is possible that between these steps the user will want to add any screening steps, or adjustments to the calibrations. PyGlider does not provide those steps, but is designed so they are easy to add. diff --git a/environment.yml b/environment.yml index 4c73e6a5..31db1a8b 100644 --- a/environment.yml +++ b/environment.yml @@ -9,9 +9,9 @@ dependencies: - dask - netcdf4 - gsw + - polars>=1.1 - scipy - bitstring - pooch - pip: - dbdreader - - polars diff --git a/pyglider/_version.py b/pyglider/_version.py index cba8e594..0fd13181 100644 --- a/pyglider/_version.py +++ b/pyglider/_version.py @@ -1 +1 @@ -__version__ = '0.0.4' \ No newline at end of file +__version__ = '0.0.7' \ No newline at end of file diff --git a/pyglider/ncprocess.py b/pyglider/ncprocess.py index 89bb097d..7a7f0bed 100644 --- a/pyglider/ncprocess.py +++ b/pyglider/ncprocess.py @@ -14,7 +14,7 @@ _log = logging.getLogger(__name__) -def extract_timeseries_profiles(inname, outdir, deploymentyaml): +def extract_timeseries_profiles(inname, outdir, deploymentyaml, force=False): """ Extract and save each profile from a timeseries netCDF. @@ -29,16 +29,18 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): deploymentyaml : str or Path location of deployment yaml file for the netCDF file. This should be the same yaml file that was used to make the timeseries file. + + force : bool, default False + Force an overwite even if profile netcdf already exists """ try: os.mkdir(outdir) except FileExistsError: pass - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) - meta = deployment['metadata'] + deployment = utils._get_deployment(deploymentyaml) + meta = deployment['metadata'] with xr.open_dataset(inname) as ds: _log.info('Extracting profiles: opening %s', inname) profiles = np.unique(ds.profile_index) @@ -49,7 +51,7 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): dss = ds.isel(time=ind) outname = outdir + '/' + utils.get_file_id(dss) + '.nc' _log.info('Checking %s', outname) - if not os.path.exists(outname): + if force or (not os.path.exists(outname)): # this is the id for the whole file, not just this profile.. dss['trajectory'] = utils.get_file_id(ds).encode() trajlen = len(utils.get_file_id(ds).encode()) @@ -68,16 +70,32 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): dss['v'] = dss.water_velocity_northward.mean() dss['v'].attrs = profile_meta['v'] elif 'u' in profile_meta: - dss['u'] = profile_meta['u'].get('_FillValue', np.NaN) + dss['u'] = profile_meta['u'].get('_FillValue', np.nan) dss['u'].attrs = profile_meta['u'] - dss['v'] = profile_meta['v'].get('_FillValue', np.NaN) + dss['v'] = profile_meta['v'].get('_FillValue', np.nan) dss['v'].attrs = profile_meta['v'] + else: + dss['u'] = np.nan + dss['v'] = np.nan - dss['profile_id'] = np.array(p*1.0) + dss['profile_id'] = np.int32(p) dss['profile_id'].attrs = profile_meta['profile_id'] + if '_FillValue' not in dss['profile_id'].attrs: + dss['profile_id'].attrs['_FillValue'] = -1 + dss['profile_id'].attrs['valid_min'] = np.int32(dss['profile_id'].attrs['valid_min']) + dss['profile_id'].attrs['valid_max'] = np.int32(dss['profile_id'].attrs['valid_max']) dss['profile_time'] = dss.time.mean() dss['profile_time'].attrs = profile_meta['profile_time'] + # BUG: If 'units' does not exist, 'calendar' may fail to be removed + # Remove attribute removal as this is cleaned up last. + ''' + try: + del dss.profile_time.attrs['units'] + del dss.profile_time.attrs['calendar'] + except KeyError: + pass + ''' dss['profile_lon'] = dss.longitude.mean() dss['profile_lon'].attrs = profile_meta['profile_lon'] dss['profile_lat'] = dss.latitude.mean() @@ -85,7 +103,7 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): dss['lat'] = dss['latitude'] dss['lon'] = dss['longitude'] - dss['platform'] = np.NaN + dss['platform'] = np.int32(1) comment = (meta['glider_model'] + ' operated by ' + meta['institution']) dss['platform'].attrs['comment'] = comment @@ -96,40 +114,62 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): meta['glider_model'] + dss['platform'].attrs['id']) dss['platform'].attrs['type'] = 'platform' dss['platform'].attrs['wmo_id'] = meta['wmo_id'] + if '_FillValue' not in dss['platform'].attrs: + dss['platform'].attrs['_FillValue'] = -1 - dss['lat_uv'] = np.NaN + dss['lat_uv'] = np.nan dss['lat_uv'].attrs = profile_meta['lat_uv'] - dss['lon_uv'] = np.NaN + dss['lon_uv'] = np.nan dss['lon_uv'].attrs = profile_meta['lon_uv'] - dss['time_uv'] = np.NaN + dss['time_uv'] = np.nan dss['time_uv'].attrs = profile_meta['time_uv'] - dss['instrument_ctd'] = np.NaN + dss['instrument_ctd'] = np.int32(1.0) dss['instrument_ctd'].attrs = profile_meta['instrument_ctd'] + if '_FillValue' not in dss['instrument_ctd'].attrs: + dss['instrument_ctd'].attrs['_FillValue'] = -1 dss.attrs['date_modified'] = str(np.datetime64('now')) + 'Z' - # ancillary variables:: + # ancillary variables: link and create with values of 2. If + # we dont' want them all 2, then create these variables in the + # time series to_fill = ['temperature', 'pressure', 'conductivity', - 'salinity', 'density', 'lon', 'lat', 'depth'] + 'salinity', 'density', 'lon', 'lat', 'depth'] for name in to_fill: - dss[name].attrs['ancillary_variables'] = name + '_qc' + qcname = name + '_qc' + dss[name].attrs['ancillary_variables'] = qcname + if qcname not in dss.keys(): + dss[qcname] = ('time', 2 * np.ones(len(dss[name]), np.int8)) + dss[qcname].attrs = utils.fill_required_qcattrs({}, name) + # 2 is "not eval" # outname = outdir + '/' + utils.get_file_id(dss) + '.nc' _log.info('Writing %s', outname) - timeunits = 'seconds since 1970-01-01T00:00:00Z' - timecalendar = 'gregorian' - dss.to_netcdf(outname, encoding={'time': {'units': timeunits, - 'calendar': timecalendar}, - 'profile_time': - {'units': timeunits}}) - - # add traj_strlen using bare ntcdf to make IOOS happy + + default_timeunits = 'seconds since 1990-01-01T00:00:00Z' + default_timecalendar = 'gregorian' + + # BUG: If '_FillValue' does not exist, 'units' may fail to be removed + # Remove in favor of final scan of metadata + ''' + try: + del dss.profile_time.attrs['_FillValue'] + del dss.profile_time.attrs['units'] + except KeyError: + pass + ''' + + (dss, encoding) = resync_metadata_and_encoding( + dss, deployment, default_timeunits, default_timecalendar) + + dss.to_netcdf(outname, encoding=encoding) + + # add traj_strlen using bare netcdf to make IOOS happy with netCDF4.Dataset(outname, 'r+') as nc: nc.renameDimension('string%d' % trajlen, 'traj_strlen') - -def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): +def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1, starttime='1970-01-01'): """ Turn a timeseries netCDF file into a vertically gridded netCDF. @@ -160,11 +200,12 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): except FileExistsError: pass - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) + deployment = utils._get_deployment(deploymentyaml) + profile_meta = deployment['profile_variables'] - ds = xr.open_dataset(inname) + ds = xr.open_dataset(inname, decode_times=True) + ds = ds.where(ds.time > np.datetime64(starttime), drop=True) _log.info(f'Working on: {inname}') _log.debug(str(ds)) _log.debug(str(ds.time[0])) @@ -174,18 +215,25 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): profiles = [p for p in profiles if (~np.isnan(p) and not (p % 1) and (p > 0))] profile_bins = np.hstack((np.array(profiles) - 0.5, [profiles[-1]+0.5])) - + _log.debug(profile_bins) Nprofiles = len(profiles) _log.info(f'Nprofiles {Nprofiles}') depth_bins = np.arange(0, 1100.1, dz) depths = depth_bins[:-1] + 0.5 - + xdimname = 'time' dsout = xr.Dataset( coords={'depth': ('depth', depths), - 'profile': ('time', profiles)}) + 'profile': (xdimname, profiles)}) + dsout['depth'].attrs = {'units': 'm', + 'long_name': 'Depth', + 'standard_name': 'depth', + 'positive': 'down', + 'coverage_content_type': 'coordinate', + 'comment': 'center of depth bins'} + ds['time_1970'] = ds.temperature.copy() - ds['time_1970'].values = ds.time.values.astype(np.float64)/1e9 - for td in ('time_1970', 'longitude', 'latitude'): + ds['time_1970'].values = ds.time.values.astype(np.float64) + for td in ('longitude', 'latitude', 'time_1970'): good = np.where(~np.isnan(ds[td]) & (ds['profile_index'] % 1 == 0))[0] dat, xedges, binnumber = stats.binned_statistic( ds['profile_index'].values[good], @@ -193,24 +241,26 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): bins=[profile_bins]) if td == 'time_1970': td = 'time' - dat = dat.astype('timedelta64[s]') + np.datetime64('1970-01-01T00:00:00') + dat = dat.astype('timedelta64[ns]') + np.datetime64('1970-01-01T00:00:00') _log.info(f'{td} {len(dat)}') dsout[td] = (('time'), dat, ds[td].attrs) ds.drop('time_1970') good = np.where(~np.isnan(ds['time']) & (ds['profile_index'] % 1 == 0))[0] _log.info(f'Done times! {len(dat)}') + # This code assumes time_1970 was handled last dsout['profile_time_start'] = ( - ('time'), dat, profile_meta['profile_time_start']) + (xdimname), dat, profile_meta['profile_time_start']) dsout['profile_time_end'] = ( - ('time'), dat, profile_meta['profile_time_end']) + (xdimname), dat, profile_meta['profile_time_end']) for k in ds.keys(): - if k in ['time', 'longitude', 'latitude', 'depth'] or 'time' in k: + if k in ['time', 'profile', 'longitude', 'latitude', 'depth'] or 'time' in k: continue - _log.info('Gridding %s', k) good = np.where(~np.isnan(ds[k]) & (ds['profile_index'] % 1 == 0))[0] if len(good) <= 0: + _log.warning(f'WARNING: Insufficent data for gridding {k}') continue + _log.info('Gridding %s', k) if "average_method" in ds[k].attrs: average_method = ds[k].attrs["average_method"] ds[k].attrs["processing"] = ( @@ -230,7 +280,7 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): bins=[profile_bins, depth_bins]) _log.debug(f'dat{np.shape(dat)}') - dsout[k] = (('depth', 'time'), dat.T, ds[k].attrs) + dsout[k] = (('depth', xdimname), dat.T, ds[k].attrs) # fill gaps in data: dsout[k].values = utils.gappy_fill_vertical(dsout[k].values) @@ -246,19 +296,152 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): dsout = dsout.drop(['water_velocity_eastward', 'water_velocity_northward']) dsout.attrs = ds.attrs + dsout.attrs.pop('cdm_data_type') + # fix to be ISO parsable: + if len(dsout.attrs['deployment_start']) > 18: + dsout.attrs['deployment_start'] = dsout.attrs['deployment_start'][:19] + dsout.attrs['deployment_end'] = dsout.attrs['deployment_end'][:19] + dsout.attrs['time_coverage_start'] = dsout.attrs['time_coverage_start'][:19] + dsout.attrs['time_coverage_end'] = dsout.attrs['time_coverage_end'][:19] + # fix standard_name so they don't overlap! + try: + dsout['waypoint_latitude'].attrs.pop('standard_name') + dsout['waypoint_longitude'].attrs.pop('standard_name') + dsout['profile_time_start'].attrs.pop('standard_name') + dsout['profile_time_end'].attrs.pop('standard_name') + except: + pass + # set some attributes for cf guidance + # see H.6.2. Profiles along a single trajectory + # https://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/aphs06.html + dsout.attrs['featureType'] = 'trajectoryProfile' + dsout['profile'].attrs['cf_role'] = 'profile_id' + dsout['mission_number'] = np.int32(1) + dsout['mission_number'].attrs['cf_role'] = 'trajectory_id' + dsout = dsout.set_coords(['latitude', 'longitude', 'time']) + for k in dsout: + if k in ['profile', 'depth', 'latitude', 'longitude', 'time', 'mission_number']: + dsout[k].attrs['coverage_content_type'] = 'coordinate' + else: + dsout[k].attrs['coverage_content_type'] = 'physicalMeasurement' + outname = outdir + '/' + ds.attrs['deployment_name'] + '_grid' + fnamesuffix + '.nc' _log.info('Writing %s', outname) - timeunits = 'seconds since 1970-01-01T00:00:00Z' - dsout.to_netcdf(outname, encoding={'time': {'units': timeunits}}) + # timeunits = 'nanoseconds since 1970-01-01T00:00:00Z' + dsout.to_netcdf( + outname, + encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z', + '_FillValue': np.nan, + 'calendar': 'gregorian', + 'dtype': 'float64'}}) _log.info('Done gridding') return outname +def resync_metadata_and_encoding(dss, deployment, default_timeunits, default_timecalendar): + """ + Makes a final pass through the deployment configuration file and + updates attributes if specified. This also takes care of encoding + variables marked with standard_name="time". + + Attributes that need to be encoded are "_FillValue, calendar and units." + + Parameters + ---------- + dss : `xarray.Dataset` + Dataset for which to check attributes. + + deployment : `dict()` + Python dictionary of user provided metadata. + + default_timeunits : `str` + The default value for the `units` attribute. + + default_timecalendar : `str` + The default value for the `calendar` attribute. + + Returns + ------- + dss : `xarray.Dataset` + Adjusted attributes according to provided deployment file. + + encoding : `dict()` + Python dictionary to pass to xarray for encoding datetime + variables. + + """ + + encoding = {} + + for meta_grp in deployment.keys(): + if meta_grp == 'metadata': + # This maps to the global metadata + global_attrs = dss.attrs + for global_attr, global_value in deployment[meta_grp].items(): + if global_attr in global_attrs: + if global_attrs[global_attr] == global_value: + continue + else: + global_attrs[global_attr] == global_value + else: + # Skip glider_devices + if meta_grp == 'glider_devices': + continue + + # These groups refer to data variables + for mvar in deployment[meta_grp].keys(): + if not mvar in dss.variables: + _log.info(f"SKIPPING variable {mvar}") + continue + dattrs = dss[mvar].attrs + for mattr, mvalue in deployment[meta_grp][mvar].items(): + if mattr in dattrs: + if dattrs[mattr] == mvalue: + continue + else: + dattrs[mattr] == mvalue + + # Scan variables with dtype datetime64 ('0) & (ds.time<6.4e9), np.NaN) - ds['time'] = (('time'), (ds.time.values.astype('timedelta64[s]') + - np.datetime64('1970-01-01T00:00:00')).astype('datetime64[ns]'), attr) + ds['time'] = ds.time.where((ds.time>0) & (ds.time<6.4e9), np.nan) + # convert time to datetime64: + ds['time'] = (ds.time*1e9).astype('datetime64[ns]') + ds['time'].attrs = attr ds = utils.fill_metadata(ds, deployment['metadata'], device_data) - start = ds['time'].values[0] - end = ds['time'].values[-1] + + start = ds.time.values[0] + end = ds.time.values[0] _log.debug('Long') _log.debug(ds.longitude.values[-2000:]) - ds.attrs['deployment_start'] = str(start) - ds.attrs['deployment_end'] = str(end) + # make sure this is ISO readable.... + ds.attrs['deployment_start'] = str(start)[:18] + ds.attrs['deployment_end'] = str(end)[:18] _log.debug(ds.depth.values[:100]) _log.debug(ds.depth.values[2000:2100]) - ds = utils.get_profiles_new( - ds, filt_time=profile_filt_time, profile_min_time=profile_min_time) + if (profile_filt_time is not None) and (profile_min_time is not None): + ds = utils.get_profiles_new( + ds, filt_time=profile_filt_time, profile_min_time=profile_min_time) _log.debug(ds.depth.values[:100]) _log.debug(ds.depth.values[2000:2100]) @@ -943,8 +993,12 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, pass outname = (outdir + '/' + ds.attrs['deployment_name'] + fnamesuffix + '.nc') _log.info('writing %s', outname) + # convert time back to float64 seconds for ERDDAP etc happiness, as they won't take ns + # as a unit: ds.to_netcdf(outname, 'w', - encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z'}}) + encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z', + '_FillValue': np.nan, + 'dtype': 'float64'}}) return outname @@ -1044,6 +1098,9 @@ def parse_logfiles(files): times = [''] * 10000 gps = [''] * 10000 amph = [''] * 10000 + relcharge = np.zeros(10000) * np.nan + volts = np.zeros(10000) * np.nan + ntimes = 0 for fn in files: found_time = False @@ -1058,22 +1115,34 @@ def parse_logfiles(files): gps[ntimes - 1] = ll if found_time and "sensor:m_coulomb_amphr_total" in ll: amph[ntimes-1] = ll + if ll.startswith(' sensor:m_lithium_battery_relative_charge'): + pattern = r'=(\d+\.\d+)' + match = re.search(pattern, ll) + relcharge[ntimes-1] = float(match.group(1)) + if ll.startswith(' sensor:m_battery(volts)='): + pattern = r'=(\d+\.\d+)' + match = re.search(pattern, ll) + volts[ntimes-1] = float(match.group(1)) amph = amph[:ntimes] gps = gps[:ntimes] times = times[:ntimes] - + volts = volts[:ntimes] + relcharge = relcharge[:ntimes] # now parse them out = xr.Dataset( - coords={'time': ('surfacing', np.zeros(ntimes, dtype='datetime64[s]'))}) - out['ampH'] = ('surfacing', np.zeros(ntimes) * np.NaN) - out['lon'] = ('surfacing', np.zeros(ntimes) * np.NaN) - out['lat'] = ('surfacing', np.zeros(ntimes) * np.NaN) + coords={'time': ('surfacing', np.zeros(ntimes, dtype='datetime64[ns]'))}) + out['ampH'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['lon'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['lat'] = ('surfacing', np.zeros(ntimes) * np.nan) + # these don't need to be parsed: + out['volts'] = ('surfacing', volts[:ntimes]) + out['relcharge'] = ('surfacing', relcharge[:ntimes]) for i in range(ntimes): timestring = times[i][11:-13] try: out['time'][i] = np.datetime64( - datetime.strptime(timestring, '%a %b %d %H:%M:%S %Y')) + datetime.strptime(timestring, '%a %b %d %H:%M:%S %Y'), 'ns') st = amph[i].index('=') en = amph[i][st:].index(' ') + st out['ampH'][i] = float(amph[i][(st+1):en]) @@ -1086,5 +1155,94 @@ def parse_logfiles(files): return out +def parse_logfiles_maybe(files): + """ + Parse time, lat, lon, and amph_total from glider logfiles. + + Parameters + ---------- + files : list of strings or Paths + List of logfiles to parse. Should be sorted. + + Returns + ------- + out : xarray + xarray data set with fields time, lon, lat, ampH indexed by surfacing. + More could be added. + """ + + times = [''] * 10000 + gps = [''] * 10000 + amph = [''] * 10000 + surfacereason = '' + missionnum = [''] * 10000 + abortsegment = 0 + abortcause = 0 + + ntimes = 0 + for fn in files: + found_time = False + + with open(fn, 'r') as fin: + for l in fin: + if 'Curr Time:' in l: + times[ntimes] = l + ntimes += 1 + found_time=True + elif found_time and 'GPS Location' in l: + gps[ntimes - 1] = l + elif found_time and "sensor:m_coulomb_amphr_total" in l: + amph[ntimes-1] = l + elif found_time and "Because:" in l: + surfacereason = l + elif found_time and "MissionNum" in l: + missionnum[ntimes-1] = l + elif found_time and "abort segment:" in l: + abortsegment = l + elif found_time and "abort cause:" in l: + abortcause = l + + amph = amph[:ntimes] + gps = gps[:ntimes] + times = times[:ntimes] + missionnum = missionnum[:ntimes] + + # now parse them + out = xr.Dataset(coords={'time': ('surfacing', np.zeros(ntimes, dtype='datetime64[ns]'))}) + out['ampH'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['lon'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['lat'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['missionnum'] = ('surfacing', np.zeros(ntimes) * np.nan) + out.attrs['surfacereason'] = surfacereason + # ABORT HISTORY: last abort segment: hal_1002-2024-183-0-0 (0171.0000) + out.attrs['abortsegment'] = float(abortsegment[-11:-2]) + out.attrs['abortcause'] = abortcause + + for i in range(ntimes): + timestring = times[i][11:-13] + out['time'][i] = np.datetime64(datetime.strptime(timestring, '%a %b %d %H:%M:%S %Y'), 'ns') + try: + if '=' in amph[i]: + st = amph[i].index('=') + en = amph[i][st:].index(' ') + st + out['ampH'][i] = float(amph[i][(st+1):en]) + + # GPS Location: 4912.737 N -12357.253 E measured 110.757 secs ago + sp = gps[i].split() + out['lat'][i] = utils.nmea2deg(float(sp[2])) + out['lon'][i] = utils.nmea2deg(float(sp[4])) + + # MissionName:calvert.mi MissionNum:hal_1002-2024-183-4-41 (0175.0041) + if len(missionnum[i]) > 12: + out['missionnum'][i] = float(missionnum[i][-11:-2]) + except: + pass + + return out + + + + + __all__ = ['binary_to_rawnc', 'merge_rawnc', 'raw_to_timeseries', 'parse_gliderState', 'parse_logfiles'] diff --git a/pyglider/utils.py b/pyglider/utils.py index 146e8413..4bd9def4 100644 --- a/pyglider/utils.py +++ b/pyglider/utils.py @@ -7,9 +7,73 @@ from scipy.signal import argrelextrema import gsw import logging +from pathlib import Path +import shlex +import subprocess +import yaml + _log = logging.getLogger(__name__) +def _get_pyglider_version(): + + rc = -1 + try: + cmdString = 'git describe --all --long --dirty --abbrev=10' + cmdList = shlex.split(cmdString) + temp = subprocess.Popen(cmdList, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + stdout, stderr = temp.communicate() + stdout = "".join(stdout.decode('utf-8').split("\n")) + stderr = stderr.decode('utf-8').split("\n") + rc = temp.returncode + except Exception as e: + pass + + # On error provide the __version__ variable as the version number + if rc != 0: + from pyglider import _version + stdout = _version.__version__ + + return stdout + +def _get_netcdf_versions(): + """ + Return the C library and python library versions + of netCDF. + + Parameters + ---------- + None + + Returns + ------- + c_version : `str` + The version of the c library. + + py_version : `str` + The version of the python library. + """ + + c_version = None + py_version = None + + if not c_version: + try: + import netCDF4 + libversion = netCDF4.getlibversion() + # This may be a string like + # 4.9.2 of Oct 4 2024 15:27:22 $ + c_version = libversion.split(" ")[0] + py_version = netCDF4.__version__ + except: + pass + + if not c_version: + c_version = 'unknown' + if not py_version: + py_version = 'unknown' + + return (c_version, py_version) def get_distance_over_ground(ds): """ @@ -26,11 +90,15 @@ def get_distance_over_ground(ds): ds : `.xarray.Dataset` With ``distance_over_ground`` key. """ + good = ~np.isnan(ds.latitude + ds.longitude) - dist = gsw.distance(ds.longitude[good].values, ds.latitude[good].values)/1000 - dist = np.roll(np.append(dist, 0), 1) - dist = np.cumsum(dist) - dist = np.interp(ds.time, ds.time[good], dist) + if np.any(good): + dist = gsw.distance(ds.longitude[good].values, ds.latitude[good].values)/1000 + dist = np.roll(np.append(dist, 0), 1) + dist = np.cumsum(dist) + dist = np.interp(ds.time, ds.time[good], dist) + else: + dist = 0 * ds.latitude.values attr = {'long_name': 'distance over ground flown since mission start', 'method': 'get_distance_over_ground', 'units': 'km', @@ -56,9 +124,13 @@ def get_glider_depth(ds): """ good = np.where(~np.isnan(ds.pressure))[0] - ds['depth'] = ds.pressure * 0. - ds['depth'].values = -gsw.z_from_p(ds.pressure.values, - ds.latitude.values) + ds['depth'] = ds.pressure + try: + meanlat = ds.latitude.mean(skipna=True) + ds['depth'].values = -gsw.z_from_p(ds.pressure.values, + ds.latitude.fillna(meanlat).values) + except AttributeError: + pass # now we really want to know where it is, so interpolate: if len(good) > 0: ds['depth'].values = np.interp( @@ -69,9 +141,10 @@ def get_glider_depth(ds): 'comment': 'from science pressure and interpolated', 'instrument': 'instrument_ctd', 'observation_type': 'calulated', - 'accuracy': '1', 'precision': '2', 'resolution': '0.02', + 'accuracy': 1.0, + 'precision': 2.0, 'resolution': 0.02, 'platform': 'platform', - 'valid_min': '0', 'valid_max': '2000', + 'valid_min': 0.0, 'valid_max': 2000.0, 'reference_datum': 'surface', 'positive': 'down'} ds['depth'].attrs = attr return ds @@ -85,8 +158,11 @@ def get_profiles(ds, min_dp=10.0, inversion=3., filt_length=7, make two variables: profile_direction and profile_index; this version is good for lots of data. Less good for sparse data """ - profile = ds.pressure.values * np.NaN - direction = ds.pressure.values * np.NaN + if 'pressure' not in ds: + _log.warning('No "pressure" variable in the data set; not searching for profiles') + return ds + profile = ds.pressure.values * np.nan + direction = ds.pressure.values * np.nan pronum = 1 lastpronum = 0 @@ -150,6 +226,10 @@ def get_profiles_new(ds, min_dp=10.0, filt_time=100, profile_min_time=300): Minimum time length of profile in s. """ + if 'pressure' not in ds: + _log.warning('No "pressure" variable in the data set; not searching for profiles') + return ds + profile = ds.pressure.values * 0 direction = ds.pressure.values * 0 pronum = 1 @@ -218,7 +298,6 @@ def get_profiles_new(ds, min_dp=10.0, filt_time=100, profile_min_time=300): direction[ins] = -1 pronum += 1 - _log.debug('Doing this...') attrs = collections.OrderedDict([ ('long_name', 'profile index'), ('units', '1'), @@ -300,15 +379,16 @@ def get_derived_eos_raw(ds): ('method', 'get_derived_eos_raw'), ('observation_type', 'calulated'), ('instrument', 'instrument_ctd'), - ('valid_max', '40.0'), - ('valid_min', '0.0'), - ('accuracy', '0.01'), - ('precision', '0.01'), - ('resolution', '0.001')]) + ('valid_max', 40.0), + ('valid_min', 0.0), + ('accuracy', 0.01), + ('precision', 0.01), + ('resolution', 0.001)]) attrs = fill_required_attrs(attrs) ds['salinity'].attrs = attrs - sa = gsw.SA_from_SP(ds['salinity'], ds['pressure'], ds['longitude'], - ds['latitude']) + long = ds.longitude.fillna(ds.longitude.mean(skipna=True)) + lat = ds.latitude.fillna(ds.latitude.mean(skipna=True)) + sa = gsw.SA_from_SP(ds['salinity'], ds['pressure'], long, lat) ct = gsw.CT_from_t(sa, ds['temperature'], ds['pressure']) ds['potential_density'] = (('time'), 1000 + gsw.density.sigma0(sa, ct).values) attrs = collections.OrderedDict([ @@ -320,9 +400,9 @@ def get_derived_eos_raw(ds): ('method', 'get_derived_eos_raw'), ('observation_type', 'calulated'), ('instrument', 'instrument_ctd'), - ('accuracy', '0.01'), - ('precision', '0.01'), - ('resolution', '0.001') + ('accuracy', 0.01), + ('precision', 0.01), + ('resolution', 0.001) ]) attrs = fill_required_attrs(attrs) ds['potential_density'].attrs = attrs @@ -338,11 +418,11 @@ def get_derived_eos_raw(ds): ('sources', 'salinity temperature pressure'), ('instrument', 'instrument_ctd'), ('method', 'get_derived_eos_raw'), - ('valid_min', '1000.0'), - ('valid_max', '1040.0'), - ('accuracy', '0.01'), - ('precision', '0.01'), - ('resolution', '0.001') + ('valid_min', 990.0), + ('valid_max', 1040.0), + ('accuracy', 0.01), + ('precision', 0.01), + ('resolution', 0.001) ]) attrs = fill_required_attrs(attrs) ds['density'].attrs = attrs @@ -357,9 +437,9 @@ def get_derived_eos_raw(ds): ('observation_type', 'calulated'), ('method', 'get_derived_eos_raw'), ('instrument', 'instrument_ctd'), - ('accuracy', '0.002'), - ('precision', '0.001'), - ('resolution', '0.0001') + ('accuracy', 0.002), + ('precision', 0.001), + ('resolution', 0.0001) ]) attrs = fill_required_attrs(attrs) ds['potential_temperature'].attrs = attrs @@ -390,6 +470,22 @@ def fill_required_attrs(attrs): return attrs +def fill_required_qcattrs(attrs, varname): + required = { + "units": "1", + "flag_values": np.array([1, 2, 3, 4, 9], dtype=np.int8), + "valid_min": np.int8(1), + "valid_max": np.int8(9), + "flag_meanings": "PASS NOT_EVALUATED SUSPECT FAIL MISSING", + "standard_name": "quality_flag", + "long_name": f"Initial flag for {varname}" + } + for k in required.keys(): + if not (k in attrs.keys()): + attrs[k] = required[k] + return attrs + + def get_file_id(ds): """ Make a file id for a Dataset @@ -440,19 +536,30 @@ def fill_metadata(ds, metadata, sensor_data): """ good = ~np.isnan(ds.latitude.values + ds.longitude.values) - ds.attrs['geospatial_lat_max'] = np.max(ds.latitude.values[good]) - ds.attrs['geospatial_lat_min'] = np.min(ds.latitude.values[good]) - ds.attrs['geospatial_lon_max'] = np.max(ds.longitude.values[good]) - ds.attrs['geospatial_lon_min'] = np.min(ds.longitude.values[good]) + if np.any(good): + ds.attrs['geospatial_lat_max'] = np.max(ds.latitude.values[good]) + ds.attrs['geospatial_lat_min'] = np.min(ds.latitude.values[good]) + ds.attrs['geospatial_lon_max'] = np.max(ds.longitude.values[good]) + ds.attrs['geospatial_lon_min'] = np.min(ds.longitude.values[good]) + else: + ds.attrs['geospatial_lat_max'] = np.nan + ds.attrs['geospatial_lat_min'] = np.nan + ds.attrs['geospatial_lon_max'] = np.nan + ds.attrs['geospatial_lon_min'] = np.nan + ds.attrs['geospatial_lat_units'] = 'degrees_north' ds.attrs['geospatial_lon_units'] = 'degrees_east' - ds.attrs['netcdf_version'] = '4.0' # TODO get this somehow... - ds.attrs['history'] = 'CPROOF glider toolbox version: pre-tag' + (netcdf_c_version, netcdf_py_version) = _get_netcdf_versions() + ds.attrs['netcdf_c_version'] = netcdf_c_version + ds.attrs['netcdf_py_version'] = netcdf_py_version + pyglider_version = _get_pyglider_version() + ds.attrs['history'] = f'CPROOF glider toolbox version: {pyglider_version}' for k, v in metadata.items(): ds.attrs[k] = v - ds.attrs['featureType'] = 'timeseries' + ds.attrs['featureType'] = 'trajectory' ds.attrs['cdm_data_type'] = 'Trajectory' - ds.attrs['Conventions'] = 'CF-1.6' + ds.attrs['Conventions'] = 'CF-1.8' + ds.attrs['standard_name_vocabulary'] = 'CF STandard Name Table v72' ds.attrs['date_created'] = str(np.datetime64('now')) + 'Z' ds.attrs['date_issued'] = str(np.datetime64('now')) + 'Z' ds.attrs['date_modified'] = " " @@ -475,7 +582,7 @@ def fill_metadata(ds, metadata, sensor_data): def _zero_screen(val): - val[val == 0] = np.NaN + val[val == 0] = np.nan return val @@ -568,6 +675,7 @@ def gappy_fill_vertical(data): data[:, j][ind[0]:ind[-1]] = np.interp(int, ind, data[ind, j]) return data + def find_gaps(sample_time, timebase, maxgap): """ Return an index into *timebase* where True are times in gaps of *sample_time* larger @@ -576,18 +684,19 @@ def find_gaps(sample_time, timebase, maxgap): # figure out which sample each time in time base belongs to: time_index = np.searchsorted(sample_time, timebase, side='right') time_index = np.clip(time_index, 0, len(sample_time)-1) - + # figure out the space between sample pairs dt = np.concatenate(([0], np.diff(sample_time))) # get the gap size for each timebase data point: ddt = dt[time_index] - - # get the indices of timebase that are too large and account for the - # degenerate case when a timebase point falls directly on a sample time. - index = ~np.logical_or((ddt <= maxgap),(np.isin(timebase,sample_time))) - + + # get the indices of timebase that are too large and account for the + # degenerate case when a timebase point falls directly on a sample time. + index = ~np.logical_or((ddt <= maxgap), (np.isin(timebase,sample_time))) + return index + def _parse_gliderxml_pos(fname): """ DEPRECATED: use slocum.parse_gliderState instead @@ -666,6 +775,58 @@ def example_gridplot(filename, outname, fig.savefig(outname, dpi=dpi) +def _get_deployment(deploymentyaml): + """ + Take the list of files in *deploymentyaml* and parse them + for deployment information, with subsequent files overwriting + previous files. + """ + if isinstance(deploymentyaml, str): + deploymentyaml = [deploymentyaml,] + deployment = {} + for nn, d in enumerate(deploymentyaml): + with open(d) as fin: + deployment_ = yaml.safe_load(fin) + for k in deployment_: + deployment[k] = deployment_[k] + + return deployment + + +def _any_newer(dirname, filename): + """ + Check if any files in dirname are newer than filename + """ + filename = Path(filename) + dirname = Path(dirname) + print(filename, filename.exists()) + if not filename.exists(): + return True + + mod_time = filename.stat().st_mtime + is_newer = False + for file_path in dirname.iterdir(): + if file_path.is_file(): + if file_path.stat().st_mtime > mod_time: + is_newer = True + break + + return is_newer + + +def _get_glider_name_slocum(current_directory): + glider = current_directory.parts[-2] + mission = current_directory.parts[-1] + print(f'Glider {glider} and mission: {mission}') + slocum_glider = glider[4:] + if slocum_glider[-4:-3].isnumeric(): + slocum_glider = slocum_glider[:-4] + '_' + slocum_glider[-4:] + else: + slocum_glider = slocum_glider[:-3] + '_' + slocum_glider[-3:] + + return glider, mission, slocum_glider + + __all__ = ['get_distance_over_ground', 'get_glider_depth', 'get_profiles_new', 'get_derived_eos_raw', "fill_metadata", "nmea2deg", "gappy_fill_vertical", "oxygen_concentration_correction"] diff --git a/requirements.txt b/requirements.txt index a15c0917..a46e8aef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ numpy pooch scipy xarray -polars>0.16 +polars>=1.1 diff --git a/setup.py b/setup.py index 73946945..6f7ee007 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,8 @@ "gsw", "scipy", "bitstring", - "pooch" + "pooch", + "polars" ], license='Apache', extras_require={ @@ -25,5 +26,7 @@ "docs": ["pydata-sphinx-theme", "numpydoc", "autoapi", "myst-parser", "sphinx"] }, - zip_safe=True + zip_safe=True, + long_description=open('README.md').read(), + long_description_content_type='text/markdown', ) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..432608b0 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,5 @@ +# README + +Sample data is kept in a separate git repository + +https://github.com/c-proof/pyglider-example-data diff --git a/tests/_copyresultstoexpected.py b/tests/_copyresultstoexpected.py new file mode 100644 index 00000000..7dd1b773 --- /dev/null +++ b/tests/_copyresultstoexpected.py @@ -0,0 +1,17 @@ +""" +If we run the tests and decide we want all the test results to be copied over, this is faster... +""" +from pathlib import Path +from shutil import copy + +todo = {'example-data/example-seaexplorer/L0-timeseries-test/dfo-eva035-20190718.nc': + 'expected/example-seaexplorer/L0-timeseries', + 'example-data/example-seaexplorer-raw/L0-timeseries-test/dfo-bb046-20200908.nc': 'expected/example-seaexplorer-raw/L0-timeseries', + 'example-data/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc': 'expected/example-slocum/L0-timeseries', + 'example-data/example-slocum-littleendian/L0-timeseries-test/dfo-maria997-20220614.nc': 'expected/example-slocum-littleendian/L0-timeseries' + + } + +for td in todo: + copy(td, todo[td]) + diff --git a/tests/environment.yml b/tests/environment.yml index 5edbc773..c3e117b5 100644 --- a/tests/environment.yml +++ b/tests/environment.yml @@ -11,10 +11,12 @@ dependencies: - gsw - scipy - bitstring + - polars>=1.1 - pytest - pytest-cov - pooch - matplotlib + - compliance-checker + - cc-plugin-glider - pip: - dbdreader - - polars diff --git a/tests/example-data/example-seaexplorer-raw/deployment.yml b/tests/example-data/example-seaexplorer-raw/deployment.yml index ed821309..1809b97c 100644 --- a/tests/example-data/example-seaexplorer-raw/deployment.yml +++ b/tests/example-data/example-seaexplorer-raw/deployment.yml @@ -104,8 +104,8 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 longitude: @@ -120,29 +120,29 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 heading: source: Heading long_name: glider heading angle standard_name: platform_orientation - units: rad + units: degrees coordinates: time depth latitude longitude pitch: source: Pitch long_name: glider pitch angle standard_name: platform_pitch_angle - units: rad + units: degrees coordinates: time depth latitude longitude roll: source: Roll long_name: glider roll angle standard_name: platform_roll_angle - units: rad + units: degrees coordinates: time depth latitude longitude # data parameters @@ -153,12 +153,12 @@ netcdf_variables: units: S m-1 coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "0." - valid_max: "10." + valid_min: 0 + valid_max: 10 observation_type: "measured" - accuracy: "0.0003" - precision: "0.0001" - resolution: "0.00002" + accuracy: 0.0003 + precision: 0.0001 + resolution: 0.00002 temperature: source: GPCTD_TEMPERATURE @@ -167,12 +167,12 @@ netcdf_variables: units: Celsius coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "-5" - valid_max: "50" + valid_min: -5 + valid_max: 50 observation_type: "measured" - accuracy: "0.002" - precision: "0.001" - resolution: "0.0002" + accuracy: 0.002 + precision: 0.001 + resolution: 0.0002 pressure: source: GPCTD_PRESSURE @@ -180,15 +180,15 @@ netcdf_variables: standard_name: sea_water_pressure units: dbar coordinates: time depth latitude longitude - valid_min: "0" - valid_max: "2000" + valid_min: 0 + valid_max: 2000 positive: "down" reference_datum: "sea-surface" instrument: "instrument_ctd" observation_type: "measured" - accuracy: "1" - precision: "2" - resolution: "0.02" + accuracy: 1 + precision: 2 + resolution: 0.02 comment: "ctd pressure sensor" # optics: @@ -231,8 +231,8 @@ profile_variables: profile_id: comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment. long_name: 'Profile ID' - valid_max: '2147483647' - valid_min: '1' + valid_max: 2147483647 + valid_min: 1 profile_time: comment: Timestamp corresponding to the mid-point of the profile @@ -265,8 +265,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 profile_lon: comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile @@ -275,8 +275,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 u: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -285,8 +285,8 @@ profile_variables: platform: platform standard_name: eastward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 v: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -295,8 +295,8 @@ profile_variables: platform: platform standard_name: northward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 lon_uv: comment: Not computed @@ -305,8 +305,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 lat_uv: comment: Not computed @@ -315,8 +315,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 time_uv: comment: Not computed @@ -329,7 +329,7 @@ profile_variables: instrument_ctd: comment: pumped CTD calibration_date: "2018-11-02" - calibration_report: + calibration_report: factory_calibrated: "yes" long_name: Seabird Glider Payload CTD make_model: Seabird GPCTD diff --git a/tests/example-data/example-seaexplorer/deploymentRealtime.yml b/tests/example-data/example-seaexplorer/deploymentRealtime.yml index 50a7cf9a..3893c68e 100644 --- a/tests/example-data/example-seaexplorer/deploymentRealtime.yml +++ b/tests/example-data/example-seaexplorer/deploymentRealtime.yml @@ -100,8 +100,8 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 longitude: @@ -116,29 +116,29 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 heading: source: Heading long_name: glider heading angle standard_name: platform_orientation - units: rad + units: degrees coordinates: time depth latitude longitude pitch: source: Pitch long_name: glider pitch angle standard_name: platform_pitch_angle - units: rad + units: degrees coordinates: time depth latitude longitude roll: source: Roll long_name: glider roll angle standard_name: platform_roll_angle - units: rad + units: degrees coordinates: time depth latitude longitude # data parameters @@ -149,12 +149,12 @@ netcdf_variables: units: S m-1 coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "0." - valid_max: "10." + valid_min: 0 + valid_max: 10 observation_type: "measured" - accuracy: "0.0003" - precision: "0.0001" - resolution: "0.00002" + accuracy: 0.0003 + precision: 0.0001 + resolution: 0.00002 temperature: source: GPCTD_TEMPERATURE @@ -163,12 +163,12 @@ netcdf_variables: units: Celsius coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "-5" - valid_max: "50" + valid_min: -5 + valid_max: 50 observation_type: "measured" - accuracy: "0.002" - precision: "0.001" - resolution: "0.0002" + accuracy: 0.002 + precision: 0.001 + resolution: 0.0002 pressure: source: GPCTD_PRESSURE @@ -176,15 +176,15 @@ netcdf_variables: standard_name: sea_water_pressure units: dbar coordinates: time depth latitude longitude - valid_min: "0" - valid_max: "2000" + valid_min: 0 + valid_max: 2000 positive: "down" reference_datum: "sea-surface" instrument: "instrument_ctd" observation_type: "measured" - accuracy: "1" - precision: "2" - resolution: "0.02" + accuracy: 1 + precision: 2 + resolution: 0.02 comment: "ctd pressure sensor" # optics: @@ -244,8 +244,8 @@ profile_variables: profile_id: comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment. long_name: 'Profile ID' - valid_max: '2147483647' - valid_min: '1' + valid_max: 2147483647 + valid_min: 1 profile_time: comment: Timestamp corresponding to the mid-point of the profile @@ -275,8 +275,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 profile_lon: comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile @@ -285,8 +285,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 u: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -295,8 +295,8 @@ profile_variables: platform: platform standard_name: eastward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 v: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -305,8 +305,8 @@ profile_variables: platform: platform standard_name: northward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 lon_uv: comment: Not computed @@ -315,8 +315,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 lat_uv: comment: Not computed @@ -325,8 +325,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 time_uv: comment: Not computed diff --git a/tests/example-data/example-slocum-littleendian/deployment.yml b/tests/example-data/example-slocum-littleendian/deployment.yml index a5d258e5..346587b1 100755 --- a/tests/example-data/example-slocum-littleendian/deployment.yml +++ b/tests/example-data/example-slocum-littleendian/deployment.yml @@ -105,8 +105,8 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 longitude: @@ -120,29 +120,29 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 heading: source: m_heading long_name: glider heading angle standard_name: platform_orientation - units: rad + units: degrees coordinates: time depth latitude longitude pitch: source: m_pitch long_name: glider pitch angle standard_name: platform_pitch_angle - units: rad + units: degrees coordinates: time depth latitude longitude roll: source: m_roll long_name: glider roll angle standard_name: platform_roll_angle - units: rad + units: degrees coordinates: time depth latitude longitude # profile info: @@ -168,12 +168,12 @@ netcdf_variables: units: S m-1 coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "0." - valid_max: "10." + valid_min: 0 + valid_max: 10 observation_type: "measured" - accuracy: "0.0003" + accuracy: 0.0003 precision: " " - resolution: "0.0001" + resolution: 0.0001 temperature: source: sci_water_temp @@ -182,12 +182,12 @@ netcdf_variables: units: Celsius coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "-5" - valid_max: "50" + valid_min: -5 + valid_max: 50 observation_type: "measured" - accuracy: "0.002" + accuracy: 0.002 precision: " " - resolution: "0.00005" + resolution: 0.00005 pressure: source: sci_water_pressure @@ -196,15 +196,15 @@ netcdf_variables: units: dbar coordinates: time depth latitude longitude conversion: bar2dbar - valid_min: "0" - valid_max: "2000" + valid_min: 0 + valid_max: 2000 positive: "down" reference_datum: "sea-surface" instrument: "instrument_ctd" observation_type: "measured" - accuracy: "0.5" + accuracy: 0.5 precision: " " - resolution: "0.01" + resolution: 0.01 comment: "ctd pressure sensor" # optics: @@ -247,8 +247,8 @@ profile_variables: profile_id: comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment. long_name: 'Profile ID' - valid_max: '2147483647' - valid_min: '1' + valid_max: 2147483647 + valid_min: 1 profile_time: comment: Timestamp corresponding to the mid-point of the profile @@ -281,8 +281,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 profile_lon: comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile @@ -291,8 +291,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 u: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -301,8 +301,8 @@ profile_variables: platform: platform standard_name: eastward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 v: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -311,8 +311,8 @@ profile_variables: platform: platform standard_name: northward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 lon_uv: comment: Not computed @@ -321,8 +321,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 lat_uv: comment: Not computed @@ -331,8 +331,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 time_uv: comment: Not computed diff --git a/tests/example-data/example-slocum/deploymentRealtime.yml b/tests/example-data/example-slocum/deploymentRealtime.yml index b9f2dcb9..6c000e47 100644 --- a/tests/example-data/example-slocum/deploymentRealtime.yml +++ b/tests/example-data/example-slocum/deploymentRealtime.yml @@ -32,7 +32,7 @@ metadata: license: "This data may be redistributed and used without restriction or warranty" metadata_link: "https://cproof.uvic.ca" - Metadata_Conventions: CF-1.6, Unidata Dataset Discovery v1.0 + Metadata_Conventions: CF-1.8, Unidata Dataset Discovery v1.0 naming_authority: "ca.uvic.cproof" platform_type: "Slocum Glider" processing_level: "Data provided as is with no expressed or implied @@ -44,9 +44,9 @@ metadata: publisher_url: http://cproof.uvic.ca references: cproof toolbox URL # https://www.nodc.noaa.gov/General/NODC-Archive/seanamelist.txt - sea_name: Coastal Waters of British Columbia + sea_name: Coastal Waters of Southeast Alaska and British Columbia source: Observational data from a profiling glider. - standard_name_vocabulary: CF STandard Name Table v49 + standard_name_vocabulary: CF STandard Name Table v72 summary: Manufacturer test in Saanich Inlet. transmission_system: IRRIDIUM wmo_id: "999999" @@ -88,7 +88,6 @@ netcdf_variables: units: seconds since 1970-01-01T00:00:00Z axis: T observation_type: "measured" - coordinates: time depth latitude longitude latitude: source: m_lat @@ -96,13 +95,12 @@ netcdf_variables: standard_name: latitude units: degrees_north axis: Y - coordinates: time depth latitude longitude comment: "Estimated between surface fixes" observation_type: measured platform: platform reference: WGS84 - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 longitude: @@ -111,35 +109,31 @@ netcdf_variables: standard_name: longitude units: degrees_east axis: X - coordinates: time depth latitude longitude comment: "Estimated between surface fixes" observation_type: measured platform: platform reference: WGS84 - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 heading: source: m_heading long_name: glider heading angle standard_name: platform_orientation - units: rad - coordinates: time depth latitude longitude + units: degrees pitch: source: m_pitch long_name: glider pitch angle standard_name: platform_pitch_angle - units: rad - coordinates: time depth latitude longitude + units: degrees roll: source: m_roll long_name: glider roll angle standard_name: platform_roll_angle - units: rad - coordinates: time depth latitude longitude + units: degrees # profile info: waypoint_latitude: @@ -147,14 +141,12 @@ netcdf_variables: long_name: waypoint latitude standard_name: latitude units: degree_north - coordinates: time depth latitude longitude waypoint_longitude: source: c_wpt_lon long_name: waypoint longitude standard_name: longitude units: degree_east - coordinates: time depth latitude longitude # data parameters conductivity: @@ -162,45 +154,42 @@ netcdf_variables: long_name: water conductivity standard_name: sea_water_electrical_conductivity units: S m-1 - coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "0." - valid_max: "10." + valid_min: 0. + valid_max: 10. observation_type: "measured" - accuracy: "0.0003" - precision: "0.0001" - resolution: "0.00002" + accuracy: 0.0003 + precision: 0.0001 + resolution: 0.00002 temperature: source: sci_water_temp long_name: water temperature standard_name: sea_water_temperature units: Celsius - coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "-5" - valid_max: "50" + valid_min: -5.0 + valid_max: 50.0 observation_type: "measured" - accuracy: "0.002" - precision: "0.001" - resolution: "0.0002" + accuracy: 0.002 + precision: 0.001 + resolution: 0.0002 pressure: source: sci_water_pressure long_name: water pressure standard_name: sea_water_pressure units: dbar - coordinates: time depth latitude longitude conversion: bar2dbar - valid_min: "0" - valid_max: "2000" + valid_min: 0.0 + valid_max: 2000.0 positive: "down" reference_datum: "sea-surface" instrument: "instrument_ctd" observation_type: "measured" - accuracy: "1" - precision: "2" - resolution: "0.02" + accuracy: 1.0 + precision: 2.0 + resolution: 0.02 comment: "ctd pressure sensor" @@ -210,19 +199,16 @@ netcdf_variables: long_name: chlorophyll standard_name: concentration_of_chlorophyll_in_sea_water units: mg m-3 - coordinates: time depth latitude longitude cdom: source: sci_flbbcd_cdom_units long_name: CDOM units: ppb - coordinates: time depth latitude longitude backscatter_700: source: sci_flbbcd_bb_units long_name: 700 nm wavelength backscatter units: "1" - coordinates: time depth latitude longitude # Oxygen oxygen_concentration: @@ -230,7 +216,6 @@ netcdf_variables: long_name: oxygen concentration standard_name: mole_concentration_of_dissolved_molecular_oxygen_in_sea_water units: umol l-1 - coordinates: time depth latitude longitude # derived water speed: # water_velocity_eastward: @@ -252,8 +237,8 @@ profile_variables: profile_id: comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment. long_name: 'Profile ID' - valid_max: '2147483647' - valid_min: '1' + valid_max: 2147483646 + valid_min: 1 profile_time: comment: Timestamp corresponding to the mid-point of the profile @@ -261,6 +246,8 @@ profile_variables: observation_type: calculated platform: platform standard_name: time + units: seconds since 1990-01-01T00:00:00Z + calendar: gregorian profile_time_start: comment: Timestamp corresponding to the start of the profile @@ -283,8 +270,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 profile_lon: comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile @@ -293,8 +280,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 u: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -303,8 +290,8 @@ profile_variables: platform: platform standard_name: eastward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 v: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -313,8 +300,8 @@ profile_variables: platform: platform standard_name: northward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 lon_uv: comment: Not computed @@ -323,8 +310,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 lat_uv: comment: Not computed @@ -333,8 +320,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 time_uv: comment: Not computed diff --git a/tests/example-data/example-slocum/process_deploymentRealTime.py b/tests/example-data/example-slocum/process_deploymentRealTime.py index d242949f..35db32b9 100644 --- a/tests/example-data/example-slocum/process_deploymentRealTime.py +++ b/tests/example-data/example-slocum/process_deploymentRealTime.py @@ -51,7 +51,7 @@ rawdir, l1tsdir, deploymentyaml, profile_filt_time=100, profile_min_time=300) -if False: +if True: # make profile netcdf files for ioos gdac... ncprocess.extract_timeseries_profiles(outname, profiledir, deploymentyaml) diff --git a/tests/results/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc b/tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc similarity index 77% rename from tests/results/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc rename to tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc index 55d56cdc..c923c625 100644 Binary files a/tests/results/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc and b/tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc differ diff --git a/tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc b/tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc similarity index 70% rename from tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc rename to tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc index e1597483..b57e1e9e 100644 Binary files a/tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc and b/tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc differ diff --git a/tests/results/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc b/tests/expected/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc similarity index 89% rename from tests/results/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc rename to tests/expected/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc index d76694c1..52f34321 100644 Binary files a/tests/results/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc and b/tests/expected/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc new file mode 100644 index 00000000..5a6ee51b Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc new file mode 100644 index 00000000..cccd05b5 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc new file mode 100644 index 00000000..3d4a4f20 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc new file mode 100644 index 00000000..925006c7 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc new file mode 100644 index 00000000..049c7438 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc new file mode 100644 index 00000000..f9d6262c Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc new file mode 100644 index 00000000..5f68d78e Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc new file mode 100644 index 00000000..9c88ca8e Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc new file mode 100644 index 00000000..2928cadf Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc new file mode 100644 index 00000000..4b64629a Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc new file mode 100644 index 00000000..a6e9a231 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc new file mode 100644 index 00000000..925dca12 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc new file mode 100644 index 00000000..5f019d9c Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc new file mode 100644 index 00000000..319efb8d Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc new file mode 100644 index 00000000..063072c4 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc new file mode 100644 index 00000000..92a45b5c Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc new file mode 100644 index 00000000..4d1473f4 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc new file mode 100644 index 00000000..4237187b Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc new file mode 100644 index 00000000..391072c9 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc new file mode 100644 index 00000000..4737f89d Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc new file mode 100644 index 00000000..4892f2a1 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc new file mode 100644 index 00000000..026c8254 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc new file mode 100644 index 00000000..dfd080ff Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc new file mode 100644 index 00000000..1b6affa3 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc new file mode 100644 index 00000000..b3d7befb Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc new file mode 100644 index 00000000..34d2ad0a Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc new file mode 100644 index 00000000..c3c3a30f Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc new file mode 100644 index 00000000..c6b262a3 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc new file mode 100644 index 00000000..c46e6808 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc new file mode 100644 index 00000000..15ef4224 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc new file mode 100644 index 00000000..61768374 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc new file mode 100644 index 00000000..0c9eaf6b Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc new file mode 100644 index 00000000..5af6612b Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc new file mode 100644 index 00000000..5ecb21ff Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc new file mode 100644 index 00000000..94d92d42 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc new file mode 100644 index 00000000..727b99c7 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc new file mode 100644 index 00000000..9db57b04 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc new file mode 100644 index 00000000..8ac4a7ee Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc new file mode 100644 index 00000000..20067bd5 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc new file mode 100644 index 00000000..595ec1e2 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc new file mode 100644 index 00000000..f1aacda1 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc new file mode 100644 index 00000000..2348ff82 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc new file mode 100644 index 00000000..658a62b4 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc new file mode 100644 index 00000000..811174a2 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc new file mode 100644 index 00000000..d7a40c4e Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc new file mode 100644 index 00000000..9ba7a809 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc new file mode 100644 index 00000000..a459f35f Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc new file mode 100644 index 00000000..375a3912 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc new file mode 100644 index 00000000..821d8804 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc new file mode 100644 index 00000000..bc3a2fc2 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc new file mode 100644 index 00000000..b99394aa Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc new file mode 100644 index 00000000..6029c701 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc new file mode 100644 index 00000000..f756490a Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc new file mode 100644 index 00000000..1f14a474 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc new file mode 100644 index 00000000..a37b8aaa Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc differ diff --git a/tests/results/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc b/tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc similarity index 71% rename from tests/results/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc rename to tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc index 2513d0fe..b390195f 100644 Binary files a/tests/results/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc and b/tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc differ diff --git a/tests/test_pyglider.py b/tests/test_pyglider.py index e5663ca5..3e284b09 100644 --- a/tests/test_pyglider.py +++ b/tests/test_pyglider.py @@ -4,6 +4,7 @@ import numpy as np import yaml +import pyglider.ncprocess as ncprocess import pyglider.seaexplorer as seaexplorer import pyglider.slocum as slocum @@ -24,7 +25,7 @@ # Open test data file test_data = xr.open_dataset( library_dir / - 'tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc') + 'tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc') variables = list(output.variables) @@ -53,8 +54,13 @@ def test_example_seaexplorer_metadata(): # Test that attributes match. Have to remove creation and issue dates first output.attrs.pop('date_created') output.attrs.pop('date_issued') + output.attrs.pop('history') + output.attrs.pop('netcdf_c_version') + output.attrs.pop('netcdf_py_version') test_data.attrs.pop('date_created') test_data.attrs.pop('date_issued') + test_data.attrs.pop('history') + test_data.attrs.pop('netcdf_version') assert output.attrs == test_data.attrs @@ -84,7 +90,6 @@ def test_example_seaexplorer_interp_nrt(var): # Test raw (full resolution) seaexplorer data. - rawdir = str(example_dir / 'example-seaexplorer-raw/delayed_raw/') + '/' rawncdir = str(example_dir / 'example-seaexplorer-raw/delayed_rawnc/') + '/' deploymentyaml_raw = str(example_dir / 'example-seaexplorer-raw/deployment.yml') @@ -97,7 +102,7 @@ def test_example_seaexplorer_interp_nrt(var): # Open test data file test_data_raw = xr.open_dataset( library_dir / - 'tests/results/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc') + 'tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc') @pytest.mark.parametrize("var", variables) def test_example_seaexplorer_raw(var): @@ -117,8 +122,13 @@ def test_example_seaexplorer_metadata_raw(): # Test that attributes match. Have to remove creation and issue dates first output_raw.attrs.pop('date_created') output_raw.attrs.pop('date_issued') + output_raw.attrs.pop('history') + output_raw.attrs.pop('netcdf_c_version') + output_raw.attrs.pop('netcdf_py_version') test_data_raw.attrs.pop('date_created') test_data_raw.attrs.pop('date_issued') + test_data_raw.attrs.pop('history') + test_data_raw.attrs.pop('netcdf_version') assert output_raw.attrs == test_data_raw.attrs @@ -148,135 +158,3 @@ def test_example_seaexplorer_interp_raw(var): assert np.allclose( np.array(dt0, dtype='float64'), np.array(dt1, dtype='float64')) - - -# Create an L0 timeseries from slocum data and test that the resulting netcdf is -# identical to the test data -cacdir = str(example_dir / 'example-slocum/cac/') + '/' -sensorlist = str(example_dir / 'example-slocum/dfo-rosie713_sensors.txt') -binarydir = str(example_dir / 'example-slocum/realtime_raw/') + '/' -rawdir_slocum = str(example_dir / 'example-slocum/realtime_rawnc/') + '/' -deploymentyaml_slocum = str(example_dir / 'example-slocum/deploymentRealtime.yml') -l1tsdir = str(example_dir / 'example-slocum/L0-timeseries-test/') + '/' -scisuffix = 'tbd' -glidersuffix = 'sbd' -do_direct = True - -if do_direct: - # turn *.sbd and *.tbd into timeseries netcdf files - outname_slocum = slocum.binary_to_timeseries( - binarydir, cacdir, l1tsdir, deploymentyaml_slocum, search='*.[s|t]bd', - profile_filt_time=20, profile_min_time=20) -else: - slocum.binary_to_rawnc( - binarydir, rawdir_slocum, cacdir, sensorlist, deploymentyaml_slocum, - incremental=False, scisuffix=scisuffix, glidersuffix=glidersuffix) - - slocum.merge_rawnc(rawdir_slocum, rawdir_slocum, deploymentyaml_slocum, - scisuffix=scisuffix, glidersuffix=glidersuffix) - outname_slocum = slocum.raw_to_timeseries( - rawdir_slocum, l1tsdir, deploymentyaml_slocum, - profile_filt_time=100, profile_min_time=300) - -output_slocum = xr.open_dataset(outname_slocum) -# Open test data file -test_data_slocum = xr.open_dataset( - library_dir / - 'tests/results/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc') -variables_slocum = list(output_slocum.variables) - - -def test_variables_slocum(): - test_variables = list(test_data_slocum.variables) - test_variables.sort() - variables_slocum.sort() - assert variables_slocum == test_variables - - -@pytest.mark.parametrize("var", variables_slocum) -def test_example_slocum(var): - # Test that variables and coordinates match - assert output_slocum[var].attrs == test_data_slocum[var].attrs - if var not in ['time']: - np.testing.assert_allclose(output_slocum[var].values, - test_data_slocum[var].values, rtol=1e-6) - else: - dt0 = output_slocum[var].values - np.datetime64('2000-01-01') - dt1 = test_data_slocum[var].values - np.datetime64('2000-01-01') - assert np.allclose( - np.array(dt0, dtype='float64'), - np.array(dt1, dtype='float64')) - - -def test_example_slocum_metadata(): - # Test that attributes match. Have to remove creation and issue - # dates first - output_slocum.attrs.pop('date_created') - output_slocum.attrs.pop('date_issued') - test_data_slocum.attrs.pop('date_created') - test_data_slocum.attrs.pop('date_issued') - assert output_slocum.attrs == test_data_slocum.attrs - - -# Create an L0 timeseries from slocum data and test that the resulting netcdf is -# identical to the test data -cacdir = str(example_dir / 'example-slocum-littleendian/cac/') + '/' -sensorlist = str( - example_dir / 'example-slocum-littleendian/dfo-maria997_sensors.txt') -binarydir = str( - example_dir / 'example-slocum-littleendian/realtime_raw/') + '/' -rawdir_slocum = str( - example_dir / 'example-slocum-littleendian/realtime_rawnc/') + '/' -deploymentyaml_slocum = str( - example_dir / 'example-slocum-littleendian/deployment.yml') -l1tsdir = str( - example_dir / 'example-slocum-littleendian/L0-timeseries-test/') + '/' -scisuffix = 'tbd' -glidersuffix = 'sbd' - -slocum.binary_to_rawnc( - binarydir, rawdir_slocum, cacdir, sensorlist, deploymentyaml_slocum, - incremental=True, scisuffix=scisuffix, glidersuffix=glidersuffix) -slocum.merge_rawnc(rawdir_slocum, rawdir_slocum, deploymentyaml_slocum, - scisuffix=scisuffix, glidersuffix=glidersuffix) -outname_slocum_le = slocum.raw_to_timeseries( - rawdir_slocum, l1tsdir, deploymentyaml_slocum, - profile_filt_time=400, profile_min_time=100) -output_slocum_le = xr.open_dataset(outname_slocum_le) -# Open test data file -test_data_slocum_le = xr.open_dataset( - library_dir / - ('tests/results/example-slocum-littleendian/' + - 'L0-timeseries/dfo-maria997-20220614.nc')) -variables_slocum_le = list(output_slocum.variables) - - -def test_variables_slocum_littleendian(): - test_variables = list(test_data_slocum_le.variables) - test_variables.sort() - variables_slocum.sort() - assert variables_slocum == test_variables - - -@pytest.mark.parametrize("var", variables_slocum) -def test_example_slocum_littleendian(var): - # Test that variables and coordinates match - assert output_slocum_le[var].attrs == test_data_slocum_le[var].attrs - if var not in ['time']: - np.testing.assert_allclose(output_slocum_le[var].values, - test_data_slocum_le[var].values, rtol=1e-6) - else: - dt0 = output_slocum_le[var].values - np.datetime64('2000-01-01') - dt1 = test_data_slocum_le[var].values - np.datetime64('2000-01-01') - assert np.allclose( - np.array(dt0, dtype='float64'), - np.array(dt1, dtype='float64')) - - -def test_example_slocum_littleendian_metadata(): - # Test that attributes match. Have to remove creation and issue dates first - output_slocum_le.attrs.pop('date_created') - output_slocum_le.attrs.pop('date_issued') - test_data_slocum_le.attrs.pop('date_created') - test_data_slocum_le.attrs.pop('date_issued') - assert output_slocum_le.attrs == test_data_slocum_le.attrs diff --git a/tests/test_seaexplorer.py b/tests/test_seaexplorer.py index 4519a717..4eb3d943 100644 --- a/tests/test_seaexplorer.py +++ b/tests/test_seaexplorer.py @@ -58,12 +58,12 @@ def test_merge_rawnc(): result_default = seaexplorer.merge_parquet( 'tests/data/realtime_rawnc/', 'tests/data/realtime_rawnc/', - example_dir / 'example-seaexplorer/deploymentRealtime.yml') + str(example_dir / 'example-seaexplorer/deploymentRealtime.yml')) result_sub = seaexplorer.merge_parquet( 'tests/data/realtime_rawnc/', 'tests/data/realtime_rawnc/', - example_dir / 'example-seaexplorer/deploymentRealtime.yml', + str(example_dir / 'example-seaexplorer/deploymentRealtime.yml'), kind='sub') assert result_default is False assert result_sub is True @@ -91,11 +91,11 @@ def test_raw_to_timeseries(): with pytest.raises(FileNotFoundError) as missing_file_exc: result_default = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/', 'tests/data/l0-profiles/', - example_dir / 'example-seaexplorer/deploymentRealtime.yml', + str(example_dir / 'example-seaexplorer/deploymentRealtime.yml'), ) result_sub = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/', 'tests/data/l0-profiles/', - example_dir / 'example-seaexplorer/deploymentRealtime.yml', + str(example_dir / 'example-seaexplorer/deploymentRealtime.yml'), kind='sub') assert 'No such file or directory' in str(missing_file_exc) assert result_sub == 'tests/data/l0-profiles/dfo-eva035-20190718.nc' @@ -114,12 +114,12 @@ def test_missing_bad_timebase(): with pytest.raises(ValueError) as bad_timebase_exc: result_bad_timebase = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/', 'tests/data/l0-profiles/', - example_dir / 'example-seaexplorer/bad_timebase.yml', + str(example_dir / 'example-seaexplorer/bad_timebase.yml'), kind='sub') with pytest.raises(ValueError) as no_timebase_exc: result_no_timebase = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/', 'tests/data/l0-profiles/', - example_dir / 'example-seaexplorer/no_timebase.yml', + str(example_dir / 'example-seaexplorer/no_timebase.yml'), kind='sub') assert "sensor not found in pld1 columns" in str(bad_timebase_exc) assert "Must specify timebase" in str(no_timebase_exc) diff --git a/tests/test_slocum.py b/tests/test_slocum.py new file mode 100644 index 00000000..a90395ef --- /dev/null +++ b/tests/test_slocum.py @@ -0,0 +1,169 @@ +import xarray as xr + +from compliance_checker.runner import ComplianceChecker, CheckSuite +import json +from pathlib import Path +import pytest +import numpy as np +import yaml + +import pyglider.ncprocess as ncprocess +import pyglider.seaexplorer as seaexplorer +import pyglider.slocum as slocum + + + +library_dir = Path(__file__).parent.parent.absolute() +example_dir = library_dir / 'tests/example-data/' + +# Create an L0 timeseries from slocum data and test that the resulting netcdf is +# identical to the test data +cacdir = example_dir / 'example-slocum/cac/' +sensorlist = str(example_dir / 'example-slocum/dfo-rosie713_sensors.txt') +binarydir = str(example_dir / 'example-slocum/realtime_raw/') + '/' +rawdir_slocum = str(example_dir / 'example-slocum/realtime_rawnc/') + '/' +deploymentyaml_slocum = str(example_dir / 'example-slocum/deploymentRealtime.yml') +tsdir = str(example_dir / 'example-slocum/L0-timeseries/') + '/' +scisuffix = 'tbd' +glidersuffix = 'sbd' +profiledir = str(example_dir / 'example-slocum/L0-profiles/') +do_direct = True + +# This needs to get run every time the tests are run, so do at top level: + +# turn *.sbd and *.tbd into timeseries netcdf files +outname_slocum = slocum.binary_to_timeseries(binarydir, cacdir, tsdir, deploymentyaml_slocum, + search='*.[s|t]bd', profile_filt_time=20, + profile_min_time=20) +# make profiles... +ncprocess.extract_timeseries_profiles(outname_slocum, profiledir, deploymentyaml_slocum, + force=True) + +output_slocum = xr.open_dataset(outname_slocum) +# Open test data file +test_data_slocum = xr.open_dataset( + library_dir / + 'tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc') +variables_slocum = list(output_slocum.variables) + + +def test_variables_slocum(): + test_variables = list(test_data_slocum.variables) + test_variables.sort() + variables_slocum.sort() + assert variables_slocum == test_variables + + +@pytest.mark.parametrize("var", variables_slocum) +def test_example_slocum(var): + # Test that variables and coordinates match + assert output_slocum[var].attrs == test_data_slocum[var].attrs + if var not in ['time']: + np.testing.assert_allclose(output_slocum[var].values, + test_data_slocum[var].values, rtol=1e-6) + else: + dt0 = output_slocum[var].values - np.datetime64('2000-01-01') + dt1 = test_data_slocum[var].values - np.datetime64('2000-01-01') + assert np.allclose( + np.array(dt0, dtype='float64'), + np.array(dt1, dtype='float64')) + + +def test_example_slocum_metadata(): + # Test that attributes match. Have to remove creation and issue + # dates first + output_slocum.attrs.pop('date_created') + output_slocum.attrs.pop('date_issued') + output_slocum.attrs.pop('history') + output_slocum.attrs.pop('netcdf_c_version') + output_slocum.attrs.pop('netcdf_py_version') + test_data_slocum.attrs.pop('date_created') + test_data_slocum.attrs.pop('date_issued') + test_data_slocum.attrs.pop('history') + test_data_slocum.attrs.pop('netcdf_version') + assert output_slocum.attrs == test_data_slocum.attrs + + +# test the profiles with compliance_checker... + +def test_profiles_compliant(): + # Load all available checker classes + check_suite = CheckSuite() + check_suite.load_all_available_checkers() + # Run cf and adcc checks + path = profiledir + '/dfo-rosie713-20190620T1313.nc' + checker_names = ['gliderdac', 'cf:1.8'] + verbose = 0 + criteria = 'normal' + output_filename = example_dir / 'report.json' + output_format = 'json' + """ + Inputs to ComplianceChecker.run_checker + + path Dataset location (url or file) + checker_names List of string names to run, + should match keys of checkers dict (empty list means run all) + verbose Verbosity of the output (0, 1, 2) + criteria Determines failure (lenient, normal, strict) + output_filename Path to the file for output + output_format Format of the output + + @returns If the tests failed (based on the criteria) + """ + return_value, errors = ComplianceChecker.run_checker(path, + checker_names, + verbose, + criteria, + output_filename=output_filename, + output_format=output_format) + # Open the JSON output and get the compliance scores + with open(output_filename, 'r') as fp: + cc_data = json.load(fp) + test = cc_data['gliderdac'] + assert test['high_count'] == 0 + assert test['medium_count'] == 0 + assert test['low_count'] == 0 + test = cc_data['cf:1.8'] + assert test['high_count'] == 0 + assert test['medium_count'] == 0 + assert test['low_count'] == 0 + + +def test_timeseries_compliant(): + # Load all available checker classes + check_suite = CheckSuite() + check_suite.load_all_available_checkers() + # Run cf and adcc checks + path = tsdir + '/dfo-rosie713-20190615.nc' + checker_names = ['cf:1.8'] + verbose = 0 + criteria = 'normal' + output_filename = example_dir / 'report.json' + output_format = 'json' + """ + Inputs to ComplianceChecker.run_checker + + path Dataset location (url or file) + checker_names List of string names to run, + should match keys of checkers dict (empty list means run all) + verbose Verbosity of the output (0, 1, 2) + criteria Determines failure (lenient, normal, strict) + output_filename Path to the file for output + output_format Format of the output + + @returns If the tests failed (based on the criteria) + """ + return_value, errors = ComplianceChecker.run_checker(path, + checker_names, + verbose, + criteria, + output_filename=output_filename, + output_format=output_format) + # Open the JSON output and get the compliance scores + with open(output_filename, 'r') as fp: + cc_data = json.load(fp) + test = cc_data['cf:1.8'] + assert test['high_count'] == 0 + # somehow the checker is confused by our trajectory variables. + assert test['medium_count'] == 1 + assert test['low_count'] == 0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 8fc0c861..69439e42 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -10,7 +10,7 @@ library_dir = Path(__file__).parent.parent.absolute() example_dir = library_dir / 'tests/example-data/' -test_data = xr.open_dataset(library_dir / 'tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc') +test_data = xr.open_dataset(library_dir / 'tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc') deploymentyaml = example_dir / 'example-seaexplorer-legato-flntu-arod-ad2cp/deploymentRealtime.yml' with open(deploymentyaml) as fin: deployment = yaml.safe_load(fin)