diff --git a/.gitignore b/.gitignore index a37a8c63..76707270 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ build/ *.csv *.yaml +*.json localdata/ src/access_mopper.egg-info MOPPeR_outputs diff --git a/ci/environment-3.11.yml b/ci/environment-3.11.yml index 6ef67ac2..48f766a1 100644 --- a/ci/environment-3.11.yml +++ b/ci/environment-3.11.yml @@ -6,7 +6,7 @@ channels: dependencies: - python==3.11 - xarray - - cmor + - cmor==3.9.* - numpy - dask - pyyaml diff --git a/src/access_mopper/_version.py b/src/access_mopper/_version.py index 956fbc2e..c231703c 100644 --- a/src/access_mopper/_version.py +++ b/src/access_mopper/_version.py @@ -1,4 +1,3 @@ - # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -12,12 +11,12 @@ """Git implementation of _version.py.""" import errno +import functools import os import re import subprocess import sys from typing import Any, Callable, Dict, List, Optional, Tuple -import functools def get_keywords() -> Dict[str, str]: @@ -68,12 +67,14 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate @@ -100,10 +101,14 @@ def run_command( try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) + process = subprocess.Popen( + [command] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + **popen_kwargs, + ) break except OSError as e: if e.errno == errno.ENOENT: @@ -141,15 +146,21 @@ def versions_from_parentdir( for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -212,7 +223,7 @@ def git_versions_from_keywords( # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -221,7 +232,7 @@ def git_versions_from_keywords( # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -229,32 +240,36 @@ def git_versions_from_keywords( for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') - if not re.match(r'\d', r): + if not re.match(r"\d", r): continue if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs( - tag_prefix: str, - root: str, - verbose: bool, - runner: Callable = run_command + tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command ) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. @@ -273,8 +288,7 @@ def git_pieces_from_vcs( env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=not verbose) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -282,10 +296,19 @@ def git_pieces_from_vcs( # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) + describe_out, rc = runner( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + f"{tag_prefix}[[:digit:]]*", + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -300,8 +323,7 @@ def git_pieces_from_vcs( pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") @@ -341,17 +363,16 @@ def git_pieces_from_vcs( dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -360,10 +381,12 @@ def git_pieces_from_vcs( if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -412,8 +435,7 @@ def render_pep440(pieces: Dict[str, Any]) -> str: rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -442,8 +464,7 @@ def render_pep440_branch(pieces: Dict[str, Any]) -> str: rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -604,11 +625,13 @@ def render_git_describe_long(pieces: Dict[str, Any]) -> str: def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -632,9 +655,13 @@ def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } def get_versions() -> Dict[str, Any]: @@ -648,8 +675,7 @@ def get_versions() -> Dict[str, Any]: verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass @@ -658,13 +684,16 @@ def get_versions() -> Dict[str, Any]: # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for _ in cfg.versionfile_source.split('/'): + for _ in cfg.versionfile_source.split("/"): root = os.path.dirname(root) except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None, + } try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -678,6 +707,10 @@ def get_versions() -> Dict[str, Any]: except NotThisMethod: pass - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } diff --git a/src/access_mopper/calc_seaice.py b/src/access_mopper/calc_seaice.py index fdced127..eeb8f54c 100644 --- a/src/access_mopper/calc_seaice.py +++ b/src/access_mopper/calc_seaice.py @@ -103,8 +103,10 @@ def get_grid_cell_length(self, xy): elif xy == "x": L = self.gridfile.hue / 100 # grid cell length in m (from cm) else: - raise Exception("""Need to supply value either 'x' or 'y' - for ice Transports""") + raise Exception( + """Need to supply value either 'x' or 'y' + for ice Transports""" + ) return L @@ -159,8 +161,10 @@ def transAcrossLine(self, var, i_start, i_end, j_start, j_end): return trans else: - raise Exception("""ERROR: Transport across a line needs to - be calculated for a single value of i or j""") + raise Exception( + """ERROR: Transport across a line needs to + be calculated for a single value of i or j""" + ) def lineTransports(self, tx_trans, ty_trans): """ diff --git a/src/access_mopper/configurations.py b/src/access_mopper/configurations.py index 9f8fe170..6f38ec17 100644 --- a/src/access_mopper/configurations.py +++ b/src/access_mopper/configurations.py @@ -74,7 +74,11 @@ def cmorise(self, file_paths, compound_name, cmor_dataset_json, mip_table): dim_mapping = mapping["dimensions"] axes = {dim_mapping.get(axis, axis): axis for axis in var.dims} - data = var.values + # CMOR 3.10 no longer accept NaN values as missing data. Use 1e20 instead. + # Fill Nan in data + var_fixed = var.fillna(1e20) + + data = var_fixed.values lat_axis = axes.pop("latitude") lat = ds[lat_axis].values lat_bnds = ds[ds[lat_axis].attrs["bounds"]].values @@ -136,14 +140,21 @@ def cmorise(self, file_paths, compound_name, cmor_dataset_json, mip_table): cmor_axes.append(cmor_axis) # Define CMOR variable - cmorVar = cmor.variable(cmor_name, variable_units, cmor_axes, positive=positive) + missing = 1e20 + cmorVar = cmor.variable( + cmor_name, + variable_units, + cmor_axes, + positive=positive, + missing_value=missing, + ) # Write data to CMOR cmor.write(cmorVar, data, ntimes_passed=len(time_numeric)) # Finalize and save the file - filename = cmor.close(cmorVar, file_name=True) - print("Stored in:", filename) + self.filename = cmor.close(cmorVar, file_name=True) + print("Stored in:", self.filename) cmor.close() @@ -181,7 +192,11 @@ def cmorise_ocean(self, file_paths, compound_name, cmor_dataset_json, mip_table) y = np.arange(j_axis.size, dtype="float") y_bnds = np.array([[y_ - 0.5, y_ + 0.5] for y_ in y]) - data = var.values + # CMOR 3.10 no longer accept NaN values as missing data. Use 1e20 instead. + # Fill Nan in data + var_fixed = var.fillna(1e20) + + data = var_fixed.values lat = self.supergrid.lat lat_bnds = self.supergrid.lat_bnds @@ -258,15 +273,22 @@ def cmorise_ocean(self, file_paths, compound_name, cmor_dataset_json, mip_table) cmor_axes.append(cmor_axis) # Define CMOR variable - cmorVar = cmor.variable(cmor_name, variable_units, cmor_axes, positive=positive) + missing = 1e20 + cmorVar = cmor.variable( + cmor_name, + variable_units, + cmor_axes, + positive=positive, + missing_value=missing, + ) # Write data to CMOR data = np.moveaxis(data, 0, -1) cmor.write(cmorVar, data, ntimes_passed=len(time_numeric)) # Finalize and save the file - filename = cmor.close(cmorVar, file_name=True) - print("Stored in:", filename) + self.filename = cmor.close(cmorVar, file_name=True) + print("Stored in:", self.filename) cmor.close() @@ -315,11 +337,13 @@ def cmorise(self, file_paths, compound_name, cmor_dataset_json, mip_table): j_axis = axes.pop("latitude") j_axis = ds[j_axis].values x = np.arange(i_axis.size, dtype="float") - # x_bnds = np.array([[x_ - 0.5, x_ + 0.5] for x_ in x]) y = np.arange(j_axis.size, dtype="float") - # y_bnds = np.array([[y_ - 0.5, y_ + 0.5] for y_ in y]) - data = var.values + # CMOR 3.10 no longer accept NaN values as missing data. Use 1e20 instead. + # Fill Nan in data + var_fixed = var.fillna(1e20) + + data = var_fixed.values lat = self.supergrid.lat lat_bnds = self.supergrid.lat_bnds @@ -330,13 +354,11 @@ def cmorise(self, file_paths, compound_name, cmor_dataset_json, mip_table): # Not all variable have a time component (e.g. fx, Ofx) time_axis = axes.pop("time", None) if time_axis: - time_axis = axes.pop("time") time_numeric = ds[time_axis].values time_units = ds[time_axis].attrs["units"] time_bnds = ds[ds[time_axis].attrs["bounds"]].values # TODO: Check that the calendar is the same than the one defined in the model.json # Convert if not. - # calendar = ds[time_axis].attrs["calendar"] # CMOR setup ipth = "Test" @@ -369,7 +391,6 @@ def cmorise(self, file_paths, compound_name, cmor_dataset_json, mip_table): latitude_vertices=lat_bnds, longitude_vertices=lon_bnds, ) - cmor_axes.append(grid_id) # Now, load the Omon table to set up the time axis and variable current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -381,34 +402,68 @@ def cmorise(self, file_paths, compound_name, cmor_dataset_json, mip_table): cmorTime = cmor.axis( "time", coord_vals=time_numeric, cell_bounds=time_bnds, units=time_units ) - cmor_axes.append(cmorTime) - if axes: - for axis, dim in axes.items(): - coord_vals = var[dim].values - try: - cell_bounds = var[var[dim].attrs["bounds"]].values - except KeyError: - cell_bounds = None - axis_units = var[dim].attrs["units"] - cmor_axis = cmor.axis( - axis, - coord_vals=coord_vals, - cell_bounds=cell_bounds, - units=axis_units, - ) - cmor_axes.append(cmor_axis) + # 2d data + if data.ndim == 3: + # The order of axes follows the specification in CMIP6_Omon.json. + cmor_axes.append(grid_id) + cmor_axes.append(cmorTime) + data = np.transpose(data, (1, 2, 0)) + + # 3d data + elif data.ndim == 4: + if axes: + # Get the z-axis of the ocean variable and create bounds if they are missing in the raw data. + for axis, dim in axes.items(): + coord_vals = var[dim].values + try: + # check whether the raw data includes bounds + cell_bounds = var[var[dim].attrs["bounds"]].values + except KeyError: + # check axes name + if axis == "depth_coord": + # create bounds + depth = coord_vals + bounds = np.zeros((len(depth), 2)) + bounds[1:, 0] = 0.5 * (depth[1:] + depth[:-1]) + bounds[:-1, 1] = 0.5 * (depth[1:] + depth[:-1]) + bounds[0, 0] = max( + 0.0, depth[0] - (depth[1] - depth[0]) / 2 + ) + bounds[-1, 1] = depth[-1] + (depth[-1] - depth[-2]) / 2 + cell_bounds = bounds + axis_units = "m" + else: + raise ValueError("Unexpected Axis") + # create cmor_axis for z-axis + cmor_axis = cmor.axis( + axis, + coord_vals=coord_vals, + cell_bounds=cell_bounds, + units=axis_units, + ) + # The order of axes follows the specification in CMIP6_Omon.json. + cmor_axes.append(grid_id) + cmor_axes.append(cmor_axis) + cmor_axes.append(cmorTime) + # Transpose data shape to match the order of axis + data = np.transpose(data, (2, 3, 1, 0)) # Define CMOR variable - cmorVar = cmor.variable(cmor_name, variable_units, cmor_axes, positive=positive) + missing = 1e20 + cmorVar = cmor.variable( + cmor_name, + variable_units, + cmor_axes, + positive=positive, + missing_value=missing, + ) # Write data to CMOR - data = np.moveaxis(data, 0, -1) - ntimes_passed = len(time_numeric) if time_axis else 0 - cmor.write(cmorVar, data, ntimes_passed=ntimes_passed) + cmor.write(cmorVar, data, ntimes_passed=len(time_numeric)) # Finalize and save the file - filename = cmor.close(cmorVar, file_name=True) - print("Stored in:", filename) + self.filename = cmor.close(cmorVar, file_name=True) + print("Stored in:", self.filename) cmor.close() diff --git a/src/access_mopper/mappings/Mappings_OM3_Omon.json b/src/access_mopper/mappings/Mappings_OM3_Omon.json index 4c2e1cec..d6cd1d17 100644 --- a/src/access_mopper/mappings/Mappings_OM3_Omon.json +++ b/src/access_mopper/mappings/Mappings_OM3_Omon.json @@ -122,7 +122,7 @@ "dimensions": { "time": "time", "yh": "latitude", - "xh": "longitude" + "xq": "longitude" }, "units": "N m-2", "positive": "down", @@ -203,23 +203,6 @@ } }, - "hfsifrazil": { - "dimensions": { - "time": "time", - "yh": "latitude", - "xh": "longitude" - }, - "units": "W m-2", - "positive": null, - "model_variables": [ - "hfsifrazil" - ], - "calculation": { - "type": "direct", - "formula": "hfsifrazil" - } - }, - "evs": { "dimensions": { "time": "time", @@ -325,7 +308,7 @@ "tauvo": { "dimensions": { "time": "time", - "yh": "latitude", + "yq": "latitude", "xh": "longitude" }, "units": "N m-2", @@ -354,6 +337,150 @@ "type": "direct", "formula": "sfdsi" } + }, + + "vo": { + "dimensions": { + "time": "time", + "zl": "depth_coord", + "yq": "latitude", + "xh": "longitude" + }, + "units": "m s-1", + "positive": null, + "model_variables": [ + "vo" + ], + "calculation": { + "type": "direct", + "formula": "vo" + } + }, + + "uo": { + "dimensions": { + "time": "time", + "zl": "depth_coord", + "yh": "latitude", + "xq": "longitude" + }, + "units": "m s-1", + "positive": null, + "model_variables": [ + "uo" + ], + "calculation": { + "type": "direct", + "formula": "uo" + } + }, + + "vmo": { + "dimensions": { + "time": "time", + "zl": "depth_coord", + "yq": "latitude", + "xh": "longitude" + }, + "units": "kg s-1", + "positive": null, + "model_variables": [ + "vmo" + ], + "calculation": { + "type": "direct", + "formula": "vmo" + } + }, + + "umo": { + "dimensions": { + "time": "time", + "zl": "depth_coord", + "yh": "latitude", + "xq": "longitude" + }, + "units": "kg s-1", + "positive": null, + "model_variables": [ + "umo" + ], + "calculation": { + "type": "direct", + "formula": "umo" + } + }, + + "thkcello": { + "dimensions": { + "time": "time", + "zl": "depth_coord", + "yh": "latitude", + "xh": "longitude" + }, + "units": "m", + "positive": null, + "model_variables": [ + "thkcello" + ], + "calculation": { + "type": "direct", + "formula": "thkcello" + } + }, + + "agessc": { + "dimensions": { + "time": "time", + "zl": "depth_coord", + "yh": "latitude", + "xh": "longitude" + }, + "units": "yr", + "positive": null, + "model_variables": [ + "agessc" + ], + "calculation": { + "type": "direct", + "formula": "agessc" + } + }, + + "thetao": { + "dimensions": { + "time": "time", + "zl": "depth_coord", + "yh": "latitude", + "xh": "longitude" + }, + "units": "degC", + "positive": null, + "model_variables": [ + "thetao" + ], + "calculation": { + "type": "direct", + "formula": "thetao" + } + }, + + "so": { + "dimensions": { + "time": "time", + "zl": "depth_coord", + "yh": "latitude", + "xh": "longitude" + }, + "units": "0.001", + "positive": null, + "model_variables": [ + "so" + ], + "calculation": { + "type": "direct", + "formula": "so" + } } } diff --git a/src/access_mopper/ocean_supergrid.py b/src/access_mopper/ocean_supergrid.py index 857f8f69..e1565ffe 100644 --- a/src/access_mopper/ocean_supergrid.py +++ b/src/access_mopper/ocean_supergrid.py @@ -1,3 +1,5 @@ +import os + import numpy as np import xarray as xr @@ -7,8 +9,14 @@ class Supergrid(object): # https://gist.github.com/rbeucher/b67c2b461557bc215a70017ea8dd337b def __init__(self, supergrid_file): - self.supergrid_file = supergrid_file - self.supergrid = xr.open_dataset(supergrid_file) + if isinstance(supergrid_file, list): + self.supergrid_file_list = supergrid_file + self.supergrid = xr.open_mfdataset(self.supergrid_file_list) + elif os.path.isfile(supergrid_file): + self.supergrid_file = supergrid_file + self.supergrid = xr.open_dataset(supergrid_file) + else: + raise ValueError("Couldn't find supergrid file") # T point locations self.xt = self.supergrid["x"][1::2, 1::2] diff --git a/tests/data/om3/2d/access-om3.mom6.2d.evs.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.evs.1mon.mean.1902_01.nc new file mode 100644 index 00000000..aeaa5e9c Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.evs.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.friver.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.friver.1mon.mean.1902_01.nc new file mode 100644 index 00000000..a4fdd254 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.friver.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.fsitherm.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.fsitherm.1mon.mean.1902_01.nc new file mode 100644 index 00000000..3612b662 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.fsitherm.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.hfds.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.hfds.1mon.mean.1902_01.nc new file mode 100644 index 00000000..e85f90e4 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.hfds.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.hflso.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.hflso.1mon.mean.1902_01.nc new file mode 100644 index 00000000..2de504b6 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.hflso.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.hfrainds.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.hfrainds.1mon.mean.1902_01.nc new file mode 100644 index 00000000..fc7973bf Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.hfrainds.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.hfsso.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.hfsso.1mon.mean.1902_01.nc new file mode 100644 index 00000000..1eac006d Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.hfsso.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.mlotst.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.mlotst.1mon.mean.1902_01.nc new file mode 100644 index 00000000..9c303a10 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.mlotst.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.pbo.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.pbo.1mon.mean.1902_01.nc new file mode 100644 index 00000000..7cadad89 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.pbo.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.prsn.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.prsn.1mon.mean.1902_01.nc new file mode 100644 index 00000000..304dbc8b Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.prsn.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.pso.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.pso.1mon.mean.1902_01.nc new file mode 100644 index 00000000..b4264aea Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.pso.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.rlntds.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.rlntds.1mon.mean.1902_01.nc new file mode 100644 index 00000000..ffea7fcf Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.rlntds.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.rsntds.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.rsntds.1mon.mean.1902_01.nc new file mode 100644 index 00000000..97a16fe8 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.rsntds.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.sfdsi.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.sfdsi.1mon.mean.1902_01.nc new file mode 100644 index 00000000..61612b61 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.sfdsi.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.sos.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.sos.1mon.mean.1902_01.nc new file mode 100644 index 00000000..404caa43 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.sos.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.tauuo.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.tauuo.1mon.mean.1902_01.nc new file mode 100644 index 00000000..ef43ee20 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.tauuo.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.tauvo.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.tauvo.1mon.mean.1902_01.nc new file mode 100644 index 00000000..c009aabe Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.tauvo.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.tos.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.tos.1mon.mean.1902_01.nc new file mode 100644 index 00000000..7782853d Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.tos.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.wfo.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.wfo.1mon.mean.1902_01.nc new file mode 100644 index 00000000..1a938b1b Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.wfo.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/2d/access-om3.mom6.2d.zos.1mon.mean.1902_01.nc b/tests/data/om3/2d/access-om3.mom6.2d.zos.1mon.mean.1902_01.nc new file mode 100644 index 00000000..7fde9913 Binary files /dev/null and b/tests/data/om3/2d/access-om3.mom6.2d.zos.1mon.mean.1902_01.nc differ diff --git a/tests/data/om3/3d/access-om3.mom6.3d.agessc.1mon.mean.1924_01.nc b/tests/data/om3/3d/access-om3.mom6.3d.agessc.1mon.mean.1924_01.nc new file mode 100644 index 00000000..1837bc94 Binary files /dev/null and b/tests/data/om3/3d/access-om3.mom6.3d.agessc.1mon.mean.1924_01.nc differ diff --git a/tests/data/om3/3d/access-om3.mom6.3d.so.1mon.mean.1924_01.nc b/tests/data/om3/3d/access-om3.mom6.3d.so.1mon.mean.1924_01.nc new file mode 100644 index 00000000..1ed006ee Binary files /dev/null and b/tests/data/om3/3d/access-om3.mom6.3d.so.1mon.mean.1924_01.nc differ diff --git a/tests/data/om3/3d/access-om3.mom6.3d.thetao.1mon.mean.1924_01.nc b/tests/data/om3/3d/access-om3.mom6.3d.thetao.1mon.mean.1924_01.nc new file mode 100644 index 00000000..f8d3ead6 Binary files /dev/null and b/tests/data/om3/3d/access-om3.mom6.3d.thetao.1mon.mean.1924_01.nc differ diff --git a/tests/data/om3/3d/access-om3.mom6.3d.thkcello.1mon.mean.1924_01.nc b/tests/data/om3/3d/access-om3.mom6.3d.thkcello.1mon.mean.1924_01.nc new file mode 100644 index 00000000..f9fc1e58 Binary files /dev/null and b/tests/data/om3/3d/access-om3.mom6.3d.thkcello.1mon.mean.1924_01.nc differ diff --git a/tests/data/om3/3d/access-om3.mom6.3d.umo.1mon.mean.1924_01.nc b/tests/data/om3/3d/access-om3.mom6.3d.umo.1mon.mean.1924_01.nc new file mode 100644 index 00000000..933765d3 Binary files /dev/null and b/tests/data/om3/3d/access-om3.mom6.3d.umo.1mon.mean.1924_01.nc differ diff --git a/tests/data/om3/3d/access-om3.mom6.3d.uo.1mon.mean.1924_01.nc b/tests/data/om3/3d/access-om3.mom6.3d.uo.1mon.mean.1924_01.nc new file mode 100644 index 00000000..826439da Binary files /dev/null and b/tests/data/om3/3d/access-om3.mom6.3d.uo.1mon.mean.1924_01.nc differ diff --git a/tests/data/om3/3d/access-om3.mom6.3d.vmo.1mon.mean.1924_01.nc b/tests/data/om3/3d/access-om3.mom6.3d.vmo.1mon.mean.1924_01.nc new file mode 100644 index 00000000..e4847b7f Binary files /dev/null and b/tests/data/om3/3d/access-om3.mom6.3d.vmo.1mon.mean.1924_01.nc differ diff --git a/tests/data/om3/3d/access-om3.mom6.3d.vo.1mon.mean.1924_01.nc b/tests/data/om3/3d/access-om3.mom6.3d.vo.1mon.mean.1924_01.nc new file mode 100644 index 00000000..65ea4603 Binary files /dev/null and b/tests/data/om3/3d/access-om3.mom6.3d.vo.1mon.mean.1924_01.nc differ diff --git a/tests/data/om3/supergrid/x.nc b/tests/data/om3/supergrid/x.nc new file mode 100644 index 00000000..8b1fba77 Binary files /dev/null and b/tests/data/om3/supergrid/x.nc differ diff --git a/tests/data/om3/supergrid/y.nc b/tests/data/om3/supergrid/y.nc new file mode 100644 index 00000000..6bcd6465 Binary files /dev/null and b/tests/data/om3/supergrid/y.nc differ diff --git a/tests/test_mop.py b/tests/test_mop.py index 02a80ec9..1660141f 100644 --- a/tests/test_mop.py +++ b/tests/test_mop.py @@ -1,9 +1,13 @@ +import glob import importlib.resources as resources +import os from pathlib import Path +import iris import pandas as pd import pytest -from access_mopper.configurations import ACCESS_ESM16_CMIP6 +from esmvalcore.preprocessor import cmor_check_data, cmor_check_metadata +from access_mopper.configurations import ACCESS_ESM16_CMIP6, ACCESS_OM3_CMIP6 DATA_DIR = Path(__file__).parent / "data" @@ -31,6 +35,29 @@ def model(): return model_instance +@pytest.fixture +def model_om3(): + # Create and save the om3 model + model_instance = ACCESS_OM3_CMIP6( + experiment_id="historical", + realization_index="1", + initialization_index="1", + physics_index="1", + forcing_index="1", + parent_mip_era="CMIP6", + parent_activity_id="CMIP", + parent_experiment_id="piControl", + parent_source_id="ACCESS-ESM1-5", + parent_variant_label="r1i1p1f1", + parent_time_units="days since 0101-1-1", + branch_method="standard", + branch_time_in_child=0.0, + branch_time_in_parent=0.0, + ) + model_instance.save_to_file("model_om3.json") + return model_instance + + def test_model_function(): test_file = DATA_DIR / "esm1-6/atmosphere/aiihca.pa-101909_mon.nc" assert test_file.exists(), "Test data file missing!" @@ -43,6 +70,32 @@ def load_filtered_variables(mappings): return df.index.tolist() +def load_filtered_variables_om3_2d(mappings): + # Load and filter variables from the JSON file + with resources.files("access_mopper.mappings").joinpath(mappings).open() as f: + df = pd.read_json(f, orient="index") + list_2d = [var for var in df.index if len(df.loc[var, "dimensions"]) == 3] + return list_2d + + +def load_filtered_variables_om3_3d(mappings): + # Load and filter variables from the JSON file + with resources.files("access_mopper.mappings").joinpath(mappings).open() as f: + df = pd.read_json(f, orient="index") + list_3d = [var for var in df.index if len(df.loc[var, "dimensions"]) == 4] + return list_3d + + +def esmvaltool_cmor_check(cube, cmor_name, mip, cmor_table="CMIP6", check_level=5): + # Use ESMValTool’s built-in functions cmor_check_metadata and cmor_check_data to assist with testing. + cmor_check_metadata( + cube, cmor_table, mip, short_name=cmor_name, check_level=check_level + ) + cmor_check_data( + cube, cmor_table, mip, short_name=cmor_name, check_level=check_level + ) + + @pytest.mark.parametrize( "cmor_name", load_filtered_variables("Mappings_CMIP6_Amon.json") ) @@ -55,6 +108,8 @@ def test_cmorise_CMIP6_Amon(model, cmor_name): cmor_dataset_json="model.json", mip_table="CMIP6_Amon.json", ) + test_cube = iris.load(model.filename)[0] + esmvaltool_cmor_check(test_cube, cmor_name, mip="Amon") except Exception as e: pytest.fail(f"Failed processing {cmor_name} with table CMIP6_Amon.json: {e}") @@ -71,6 +126,8 @@ def test_cmorise_CMIP6_Lmon(model, cmor_name): cmor_dataset_json="model.json", mip_table="CMIP6_Lmon.json", ) + test_cube = iris.load(model.filename)[0] + esmvaltool_cmor_check(test_cube, cmor_name, mip="Lmon") except Exception as e: pytest.fail(f"Failed processing {cmor_name} with table CMIP6_Lmon.json: {e}") @@ -87,5 +144,50 @@ def test_cmorise_CMIP6_Emon(model, cmor_name): cmor_dataset_json="model.json", mip_table="CMIP6_Emon.json", ) + test_cube = iris.load(model.filename)[0] + esmvaltool_cmor_check(test_cube, cmor_name, mip="Emon") except Exception as e: pytest.fail(f"Failed processing {cmor_name} with table CMIP6_Emon.json: {e}") + + +@pytest.mark.parametrize( + "cmor_name", load_filtered_variables_om3_2d("Mappings_OM3_Omon.json") +) +def test_cmorise_OM3_2d(model_om3, cmor_name): + file_pattern = ( + DATA_DIR / f"om3/2d/access-om3.mom6.2d.{cmor_name}.1mon.mean.1902_01.nc" + ) + try: + model_om3.supergrid = glob.glob(str(DATA_DIR / "om3/supergrid/*.nc")) + model_om3.cmorise( + file_paths=file_pattern, + compound_name="Omon." + cmor_name, + cmor_dataset_json="model_om3.json", + mip_table="CMIP6_Omon.json", + ) + test_cube = iris.load(model_om3.filename)[0] + esmvaltool_cmor_check(test_cube, cmor_name, mip="Omon") + except Exception as e: + pytest.fail(f"Failed processing {cmor_name} with table CMIP6_Omon.json: {e}") + + +@pytest.mark.parametrize( + "cmor_name", load_filtered_variables_om3_3d("Mappings_OM3_Omon.json") +) +def test_cmorise_OM3_3d(model_om3, cmor_name): + print("Working directory:", os.getcwd()) + file_pattern = ( + DATA_DIR / f"om3/3d/access-om3.mom6.3d.{cmor_name}.1mon.mean.1924_01.nc" + ) + try: + model_om3.supergrid = glob.glob(str(DATA_DIR / "om3/supergrid/*.nc")) + model_om3.cmorise( + file_paths=file_pattern, + compound_name="Omon." + cmor_name, + cmor_dataset_json="model_om3.json", + mip_table="CMIP6_Omon.json", + ) + test_cube = iris.load(model_om3.filename)[0] + esmvaltool_cmor_check(test_cube, cmor_name, mip="Omon") + except Exception as e: + pytest.fail(f"Failed processing {cmor_name} with table CMIP6_Omon.json: {e}") diff --git a/versioneer.py b/versioneer.py index 1e3753e6..52a7a176 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,4 +1,3 @@ - # Version: 0.29 """The Versioneer - like a rocketeer, but for versions. @@ -310,15 +309,15 @@ import configparser import errno +import functools import json import os import re import subprocess import sys from pathlib import Path -from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union -from typing import NoReturn -import functools +from typing import (Any, Callable, Dict, List, NoReturn, Optional, Tuple, + Union, cast) have_tomllib = True if sys.version_info >= (3, 11): @@ -367,11 +366,13 @@ def get_root() -> str: or os.path.exists(pyproject_toml) or os.path.exists(versioneer_py) ): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -384,8 +385,10 @@ def get_root() -> str: me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(my_path), versioneer_py) + ) except NameError: pass return root @@ -403,9 +406,9 @@ def get_config_from_root(root: str) -> VersioneerConfig: section: Union[Dict[str, Any], configparser.SectionProxy, None] = None if pyproject_toml.exists() and have_tomllib: try: - with open(pyproject_toml, 'rb') as fobj: + with open(pyproject_toml, "rb") as fobj: pp = tomllib.load(fobj) - section = pp['tool']['versioneer'] + section = pp["tool"]["versioneer"] except (tomllib.TOMLDecodeError, KeyError) as e: print(f"Failed to load config from {pyproject_toml}: {e}") print("Try to load it from setup.cfg") @@ -422,7 +425,7 @@ def get_config_from_root(root: str) -> VersioneerConfig: # `None` values elsewhere where it matters cfg = VersioneerConfig() - cfg.VCS = section['VCS'] + cfg.VCS = section["VCS"] cfg.style = section.get("style", "") cfg.versionfile_source = cast(str, section.get("versionfile_source")) cfg.versionfile_build = section.get("versionfile_build") @@ -450,10 +453,12 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" HANDLERS.setdefault(vcs, {})[method] = f return f + return decorate @@ -480,10 +485,14 @@ def run_command( try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) + process = subprocess.Popen( + [command] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + **popen_kwargs, + ) break except OSError as e: if e.errno == errno.ENOENT: @@ -505,7 +514,9 @@ def run_command( return stdout, process.returncode -LONG_VERSION_PY['git'] = r''' +LONG_VERSION_PY[ + "git" +] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -1250,7 +1261,7 @@ def git_versions_from_keywords( # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1259,7 +1270,7 @@ def git_versions_from_keywords( # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1267,32 +1278,36 @@ def git_versions_from_keywords( for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') - if not re.match(r'\d', r): + if not re.match(r"\d", r): continue if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs( - tag_prefix: str, - root: str, - verbose: bool, - runner: Callable = run_command + tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command ) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. @@ -1311,8 +1326,7 @@ def git_pieces_from_vcs( env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=not verbose) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1320,10 +1334,19 @@ def git_pieces_from_vcs( # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) + describe_out, rc = runner( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + f"{tag_prefix}[[:digit:]]*", + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1338,8 +1361,7 @@ def git_pieces_from_vcs( pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") @@ -1379,17 +1401,16 @@ def git_pieces_from_vcs( dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -1398,10 +1419,12 @@ def git_pieces_from_vcs( if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1479,15 +1502,21 @@ def versions_from_parentdir( for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1516,11 +1545,13 @@ def versions_from_file(filename: str) -> Dict[str, Any]: contents = f.read() except OSError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1528,8 +1559,7 @@ def versions_from_file(filename: str) -> Dict[str, Any]: def write_to_version_file(filename: str, versions: Dict[str, Any]) -> None: """Write the given version number to the given _version.py file.""" - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1561,8 +1591,7 @@ def render_pep440(pieces: Dict[str, Any]) -> str: rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1591,8 +1620,7 @@ def render_pep440_branch(pieces: Dict[str, Any]) -> str: rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1753,11 +1781,13 @@ def render_git_describe_long(pieces: Dict[str, Any]) -> str: def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1781,9 +1811,13 @@ def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1806,8 +1840,9 @@ def get_versions(verbose: bool = False) -> Dict[str, Any]: handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or bool(cfg.verbose) # `bool()` used to avoid `None` - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1861,9 +1896,13 @@ def get_versions(verbose: bool = False) -> Dict[str, Any]: if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version() -> str: @@ -1916,6 +1955,7 @@ def run(self) -> None: print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in setuptools @@ -1937,8 +1977,8 @@ def run(self) -> None: # but the build_py command is not expected to copy any files. # we override different "build_py" commands for both environments - if 'build_py' in cmds: - _build_py: Any = cmds['build_py'] + if "build_py" in cmds: + _build_py: Any = cmds["build_py"] else: from setuptools.command.build_py import build_py as _build_py @@ -1955,14 +1995,14 @@ def run(self) -> None: # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py - if 'build_ext' in cmds: - _build_ext: Any = cmds['build_ext'] + if "build_ext" in cmds: + _build_ext: Any = cmds["build_ext"] else: from setuptools.command.build_ext import build_ext as _build_ext @@ -1982,19 +2022,22 @@ def run(self) -> None: # it with an updated value if not cfg.versionfile_build: return - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) if not os.path.exists(target_versionfile): - print(f"Warning: {target_versionfile} does not exist, skipping " - "version update. This can happen if you are running build_ext " - "without first running build_py.") + print( + f"Warning: {target_versionfile} does not exist, skipping " + "version update. This can happen if you are running build_ext " + "without first running build_py." + ) return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # type: ignore + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -2015,21 +2058,27 @@ def run(self) -> None: os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: - from py2exe.setuptools_buildexe import py2exe as _py2exe # type: ignore + from py2exe.setuptools_buildexe import \ + py2exe as _py2exe # type: ignore except ImportError: - from py2exe.distutils_buildexe import py2exe as _py2exe # type: ignore + from py2exe.distutils_buildexe import \ + py2exe as _py2exe # type: ignore class cmd_py2exe(_py2exe): def run(self) -> None: @@ -2044,18 +2093,22 @@ def run(self) -> None: os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # sdist farms its file list building out to egg_info - if 'egg_info' in cmds: - _egg_info: Any = cmds['egg_info'] + if "egg_info" in cmds: + _egg_info: Any = cmds["egg_info"] else: from setuptools.command.egg_info import egg_info as _egg_info @@ -2068,7 +2121,7 @@ def find_sources(self) -> None: # Modify the filelist and normalize it root = get_root() cfg = get_config_from_root(root) - self.filelist.append('versioneer.py') + self.filelist.append("versioneer.py") if cfg.versionfile_source: # There are rare cases where versionfile_source might not be # included by default, so we must be explicit @@ -2081,18 +2134,21 @@ def find_sources(self) -> None: # We will instead replicate their final normalization (to unicode, # and POSIX-style paths) from setuptools import unicode_utils - normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') - for f in self.filelist.files] - manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') - with open(manifest_filename, 'w') as fobj: - fobj.write('\n'.join(normalized)) + normalized = [ + unicode_utils.filesys_decode(f).replace(os.sep, "/") + for f in self.filelist.files + ] + + manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") + with open(manifest_filename, "w") as fobj: + fobj.write("\n".join(normalized)) - cmds['egg_info'] = cmd_egg_info + cmds["egg_info"] = cmd_egg_info # we override different "sdist" commands for both environments - if 'sdist' in cmds: - _sdist: Any = cmds['sdist'] + if "sdist" in cmds: + _sdist: Any = cmds["sdist"] else: from setuptools.command.sdist import sdist as _sdist @@ -2114,8 +2170,10 @@ def make_release_tree(self, base_dir: str, files: List[str]) -> None: # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -2175,11 +2233,9 @@ def do_setup() -> int: root = get_root() try: cfg = get_config_from_root(root) - except (OSError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: if isinstance(e, (OSError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -2188,15 +2244,18 @@ def do_setup() -> int: print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") maybe_ipy: Optional[str] = ipy if os.path.exists(ipy): try: