From 26e3249a1edeb6a4019c5952e35e47991df9d169 Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 12:03:26 -0800 Subject: [PATCH 1/9] allow xarray 2025.12.0+ to be installed for debugging --- docs/requirements.txt | 2 +- environment.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 16b09127..be7fcde5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # Core dependencies -xarray>=2024.1.0,<=2025.9.0 +xarray>=2024.1.0 pyodbc==4.0.39 setuptools==78.1.1 pandas==2.0.3 diff --git a/environment.yml b/environment.yml index 1dd98fa0..4bd5ba3e 100644 --- a/environment.yml +++ b/environment.yml @@ -8,7 +8,7 @@ dependencies: - numpy>=1.26.2 - matplotlib>=3.8.2 - scipy>=1.11.4 - - xarray>=2024.1.0,<=2025.9.0 + - xarray>=2024.1.0 - pandas>=2.0.3 - pyodbc>=4.0.39 - pyshtools diff --git a/pyproject.toml b/pyproject.toml index b1491e5c..b293c459 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "numpy>=1.26.2", "matplotlib>=3.8.2", "scipy>=1.11.4", - "xarray>=2024.1.0,<=2025.9.0", + "xarray>=2024.1.0", "pandas>=2.0.3", "pyodbc>=4.0.39", "pypdf==6.4.0", From e38ce761272cd7f29aac8b2034ea49411ebe620b Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 12:10:24 -0800 Subject: [PATCH 2/9] fixing vertical flip issue marsformat --- bin/MarsFormat.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/bin/MarsFormat.py b/bin/MarsFormat.py index 87750127..07957307 100755 --- a/bin/MarsFormat.py +++ b/bin/MarsFormat.py @@ -805,12 +805,24 @@ def main(): else: # Standard vertical processing for other models if DS[model.pfull].values[0] != DS[model.pfull].values.min(): + # DS = DS.isel(**{model.dim_pfull: slice(None, None, -1)}) + # # Flip phalf, ak, bk: + # DS = DS.isel(**{model.dim_phalf: slice(None, None, -1)}) + # print(f"{Red}NOTE: all variables flipped along vertical dimension. " + # f"Top of the atmosphere is now index = 0") DS = DS.isel(**{model.dim_pfull: slice(None, None, -1)}) + # Flip vertical dimensions using explicit index arrays + # # This approach is more robust across xarray versions than slice(None, None, -1) + # # which can cause dimension tracking issues in xarray >= 2025.12.0 + n_pfull = DS.dims[model.dim_pfull] + DS = DS.isel(**{model.dim_pfull: list(range(n_pfull - 1, -1, -1))}) # Flip phalf, ak, bk: DS = DS.isel(**{model.dim_phalf: slice(None, None, -1)}) + n_phalf = DS.dims[model.dim_phalf] + DS = DS.isel(**{model.dim_phalf: list(range(n_phalf - 1, -1, -1))}) print(f"{Red}NOTE: all variables flipped along vertical dimension. " - f"Top of the atmosphere is now index = 0") - + f"Top of the atmosphere is now index = 0") + # Reorder dimensions print(f"{Cyan} Transposing variable dimensions to match order " f"expected in CAP") @@ -821,7 +833,10 @@ def main(): if min(DS[model.dim_lon]) < 0: tmp = np.array(DS[model.dim_lon]) tmp = np.where(tmp<0, tmp + 360, tmp) - DS[model.dim_lon] = tmp + # DS[model.dim_lon] = tmp + # Use assign_coords for robust coordinate update across xarray versions + # Direct assignment (DS[model.dim_lon] = tmp) can fail in xarray >= 2025.12.0 + DS = DS.assign_coords({model.dim_lon: tmp}) DS = DS.sortby(model.dim_lon) DS[model.lon].attrs['long_name'] = ( '(MODIFIED POST-PROCESSING) longitude' From 1e3347f8691f9093312ca4245205079e8a4863ca Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 12:23:11 -0800 Subject: [PATCH 3/9] fixing vertical flip issue marsformat --- bin/MarsFormat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/MarsFormat.py b/bin/MarsFormat.py index 07957307..a6eb70de 100755 --- a/bin/MarsFormat.py +++ b/bin/MarsFormat.py @@ -634,7 +634,9 @@ def main(): DS[model.time].attrs['units'] = 'days since 0000-00-00 00:00:00' print(f"{Cyan}Converting reference pressure to [Pa]") - DS[model.pfull] = DS[model.pfull].values*100 + # DS[model.pfull] = DS[model.pfull].values*100 + new_pfull_vals = DS[model.pfull].values * 100 + DS = DS.assign_coords({model.pfull: new_pfull_vals}) DS[model.pfull].attrs['units'] = 'Pa' # dims_list process finds dims of the variable and replaces From 7d2bd9d51e69944e293c344f9510c755cc279171 Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 12:25:00 -0800 Subject: [PATCH 4/9] fixing vertical flip issue marsformat --- bin/MarsFormat.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bin/MarsFormat.py b/bin/MarsFormat.py index a6eb70de..82cf12d6 100755 --- a/bin/MarsFormat.py +++ b/bin/MarsFormat.py @@ -808,18 +808,13 @@ def main(): # Standard vertical processing for other models if DS[model.pfull].values[0] != DS[model.pfull].values.min(): # DS = DS.isel(**{model.dim_pfull: slice(None, None, -1)}) - # # Flip phalf, ak, bk: - # DS = DS.isel(**{model.dim_phalf: slice(None, None, -1)}) - # print(f"{Red}NOTE: all variables flipped along vertical dimension. " - # f"Top of the atmosphere is now index = 0") - DS = DS.isel(**{model.dim_pfull: slice(None, None, -1)}) # Flip vertical dimensions using explicit index arrays # # This approach is more robust across xarray versions than slice(None, None, -1) # # which can cause dimension tracking issues in xarray >= 2025.12.0 n_pfull = DS.dims[model.dim_pfull] DS = DS.isel(**{model.dim_pfull: list(range(n_pfull - 1, -1, -1))}) # Flip phalf, ak, bk: - DS = DS.isel(**{model.dim_phalf: slice(None, None, -1)}) + # DS = DS.isel(**{model.dim_phalf: slice(None, None, -1)}) n_phalf = DS.dims[model.dim_phalf] DS = DS.isel(**{model.dim_phalf: list(range(n_phalf - 1, -1, -1))}) print(f"{Red}NOTE: all variables flipped along vertical dimension. " From a75fae5df9b6d3135b434fbc82df8aa8b2d3db55 Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 13:07:24 -0800 Subject: [PATCH 5/9] fixing vertical flip issue marsformat --- bin/MarsFormat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/MarsFormat.py b/bin/MarsFormat.py index 82cf12d6..d4416f25 100755 --- a/bin/MarsFormat.py +++ b/bin/MarsFormat.py @@ -811,11 +811,11 @@ def main(): # Flip vertical dimensions using explicit index arrays # # This approach is more robust across xarray versions than slice(None, None, -1) # # which can cause dimension tracking issues in xarray >= 2025.12.0 - n_pfull = DS.dims[model.dim_pfull] + n_pfull = DS.sizes[model.dim_pfull] DS = DS.isel(**{model.dim_pfull: list(range(n_pfull - 1, -1, -1))}) # Flip phalf, ak, bk: # DS = DS.isel(**{model.dim_phalf: slice(None, None, -1)}) - n_phalf = DS.dims[model.dim_phalf] + n_phalf = DS.sizes[model.dim_phalf] DS = DS.isel(**{model.dim_phalf: list(range(n_phalf - 1, -1, -1))}) print(f"{Red}NOTE: all variables flipped along vertical dimension. " f"Top of the atmosphere is now index = 0") From 6a8e00ecb821b7c09be0d04b649e4b25a82f0d13 Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 13:14:32 -0800 Subject: [PATCH 6/9] removing old email and anonymizing filepaths --- amescap/Ncdf_wrapper.py | 2 +- amescap/Script_utils.py | 2 +- bin/MarsNest.py | 2 +- bin/MarsPull.py | 2 +- docs/source/autoapi/amescap/Ncdf_wrapper/index.rst | 2 +- docs/source/autoapi/amescap/Script_utils/index.rst | 2 +- docs/source/autoapi/bin/MarsPull/index.rst | 2 +- pyproject.toml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/amescap/Ncdf_wrapper.py b/amescap/Ncdf_wrapper.py index 9443d4c5..4945a684 100644 --- a/amescap/Ncdf_wrapper.py +++ b/amescap/Ncdf_wrapper.py @@ -440,7 +440,7 @@ class Fort(object): Create a Fort object using the following:: - f=Fort('/Users/akling/test/fort.11/fort.11_0684') + f=Fort('/Users/file/test/fort.11/fort.11_0684') Public methods can be used to generate FV3-like netCDF files:: diff --git a/amescap/Script_utils.py b/amescap/Script_utils.py index cd5247fd..f7bd795d 100644 --- a/amescap/Script_utils.py +++ b/amescap/Script_utils.py @@ -554,7 +554,7 @@ def smart_reader(fNcdf, var_list, suppress_warning=False): from netCDF4 import Dataset - fNcdf = Dataset("/u/akling/FV3/00668.atmos_average_pstd.nc", "r") + fNcdf = Dataset("/u/path/to/FV3/00668.atmos_average_pstd.nc", "r") # Approach using var = fNcdf.variables["var"][:] ucomp = fNcdf.variables["ucomp"][:] diff --git a/bin/MarsNest.py b/bin/MarsNest.py index 9aa738ef..1ccb51a3 100755 --- a/bin/MarsNest.py +++ b/bin/MarsNest.py @@ -285,7 +285,7 @@ def is_child(poly_path_child,poly_path_parent): nskip=2 #skip a couple points in the topography file to speed-up plotting thick_line=0.5 #line thickness for individual cells -#TODO development map_dir='/Users/akling/Data/Mars_topo/mars_topo_mola_16.nc' +#TODO development map_dir='/Users/file/Data/Mars_topo/mars_topo_mola_16.nc' map_dir='/nobackup/rurata/FMS_MARS_data/mars_topo.nc' f=Dataset(map_dir,'r') diff --git a/bin/MarsPull.py b/bin/MarsPull.py index cead7c75..c4c4088b 100755 --- a/bin/MarsPull.py +++ b/bin/MarsPull.py @@ -199,7 +199,7 @@ def download(url, file_name): 'https://data.nas.nasa.gov/legacygcm/fv3betaout1data/03340.fixed.nc' :type url: str :param file_name: The local file_name e.g., - '/lou/la4/akling/Data/LegacyGCM_Ls000_Ls004.nc' + '/files/Data/LegacyGCM_Ls000_Ls004.nc' :type file_name: str :return: The requested file(s), downloaded and saved to the current directory. diff --git a/docs/source/autoapi/amescap/Ncdf_wrapper/index.rst b/docs/source/autoapi/amescap/Ncdf_wrapper/index.rst index 1078a321..512218af 100644 --- a/docs/source/autoapi/amescap/Ncdf_wrapper/index.rst +++ b/docs/source/autoapi/amescap/Ncdf_wrapper/index.rst @@ -50,7 +50,7 @@ Classes Create a Fort object using the following:: - f=Fort('/Users/akling/test/fort.11/fort.11_0684') + f=Fort('/Users/username/test/fort.11/fort.11_0684') Public methods can be used to generate FV3-like netCDF files:: diff --git a/docs/source/autoapi/amescap/Script_utils/index.rst b/docs/source/autoapi/amescap/Script_utils/index.rst index 6e2e6829..3bb20a96 100644 --- a/docs/source/autoapi/amescap/Script_utils/index.rst +++ b/docs/source/autoapi/amescap/Script_utils/index.rst @@ -625,7 +625,7 @@ Attributes from netCDF4 import Dataset - fNcdf = Dataset("/u/akling/FV3/00668.atmos_average_pstd.nc", "r") + fNcdf = Dataset("/u/username/FV3/00668.atmos_average_pstd.nc", "r") # Approach using var = fNcdf.variables["var"][:] ucomp = fNcdf.variables["ucomp"][:] diff --git a/docs/source/autoapi/bin/MarsPull/index.rst b/docs/source/autoapi/bin/MarsPull/index.rst index 4fdecb4a..ea271f07 100644 --- a/docs/source/autoapi/bin/MarsPull/index.rst +++ b/docs/source/autoapi/bin/MarsPull/index.rst @@ -94,7 +94,7 @@ Attributes 'https://data.nas.nasa.gov/legacygcm/fv3betaout1data/03340.fixed.nc' :type url: str :param file_name: The local file_name e.g., - '/lou/la4/akling/Data/LegacyGCM_Ls000_Ls004.nc' + '/files/Data/LegacyGCM_Ls000_Ls004.nc' :type file_name: str :return: The requested file(s), downloaded and saved to the current directory. diff --git a/pyproject.toml b/pyproject.toml index b293c459..a40e4968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.8" license = {file = "LICENSE"} authors = [ - {name = "Mars Climate Modeling Center", email = "alexandre.m.kling@nasa.gov"} + {name = "Mars Climate Modeling Center", email = "richard.a.urata@nasa.gov"} ] urls = {Homepage = "https://github.com/NASA-Planetary-Science/AmesCAP"} dependencies = [ From 498f5cfd109dba4fff0c98e65d933499ee2675a9 Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 13:17:34 -0800 Subject: [PATCH 7/9] cleaning up marsformat debug tests and comments --- bin/MarsFormat.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/bin/MarsFormat.py b/bin/MarsFormat.py index d4416f25..8f2d7653 100755 --- a/bin/MarsFormat.py +++ b/bin/MarsFormat.py @@ -807,14 +807,13 @@ def main(): else: # Standard vertical processing for other models if DS[model.pfull].values[0] != DS[model.pfull].values.min(): - # DS = DS.isel(**{model.dim_pfull: slice(None, None, -1)}) # Flip vertical dimensions using explicit index arrays - # # This approach is more robust across xarray versions than slice(None, None, -1) - # # which can cause dimension tracking issues in xarray >= 2025.12.0 + # This approach is more robust across xarray versions + # than slice(None, None, -1) which can cause dimension + # tracking issues in xarray >= 2025.12.0 n_pfull = DS.sizes[model.dim_pfull] DS = DS.isel(**{model.dim_pfull: list(range(n_pfull - 1, -1, -1))}) - # Flip phalf, ak, bk: - # DS = DS.isel(**{model.dim_phalf: slice(None, None, -1)}) + # Flip phalf: n_phalf = DS.sizes[model.dim_phalf] DS = DS.isel(**{model.dim_phalf: list(range(n_phalf - 1, -1, -1))}) print(f"{Red}NOTE: all variables flipped along vertical dimension. " @@ -830,9 +829,9 @@ def main(): if min(DS[model.dim_lon]) < 0: tmp = np.array(DS[model.dim_lon]) tmp = np.where(tmp<0, tmp + 360, tmp) - # DS[model.dim_lon] = tmp - # Use assign_coords for robust coordinate update across xarray versions - # Direct assignment (DS[model.dim_lon] = tmp) can fail in xarray >= 2025.12.0 + # Use assign_coords for robust coordinate update across + # xarray versions. Direct assignment + # (DS[model.dim_lon] = tmp) can fail in xarray >= 2025.12.0 DS = DS.assign_coords({model.dim_lon: tmp}) DS = DS.sortby(model.dim_lon) DS[model.lon].attrs['long_name'] = ( From 8de62fc6cc267927c05f7e329567a39ab8012f83 Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 13:32:37 -0800 Subject: [PATCH 8/9] phalf having issues similar to ak/bk issues with PCM. Diurn processing causing extra dims from groupby().mean() --- bin/MarsFormat.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/bin/MarsFormat.py b/bin/MarsFormat.py index 8f2d7653..f7fe95fa 100755 --- a/bin/MarsFormat.py +++ b/bin/MarsFormat.py @@ -1108,12 +1108,45 @@ def main(): f'time averaged over {nday} sols' ) + + + + # Safe phalf check for PCM files + # if model_type == 'pcm' and DS_diurn is not None and 'phalf' in DS_diurn: + # try: + # phalf_vals = DS_diurn['phalf'].values + # # Check if we have at least 2 elements + # if len(phalf_vals) > 1: + # # Extract actual values and convert to regular Python floats + # first_val = float(phalf_vals[0]) + # last_val = float(phalf_vals[-1]) + # if first_val > last_val: + # print(f"{Yellow}Warning: phalf orientation incorrect in diurn file, fixing...") + # DS_diurn['phalf'] = (DS_diurn['phalf'].dims, phalf_vals[::-1]) + # except Exception as e: + # print(f"{Yellow}Note: Could not check phalf orientation: {str(e)}") + + + + # Safe phalf check for PCM files + # During diurn processing, phalf can gain extra dimensions from + # groupby().mean() operations (similar issue to ak/bk above). + # We need to restore phalf from the original DS if it has wrong dimensions. if model_type == 'pcm' and DS_diurn is not None and 'phalf' in DS_diurn: try: + phalf_dims = DS_diurn['phalf'].dims + # Check if phalf has acquired extra dimensions (should only have 1) + if len(phalf_dims) > 1: + print(f"DEBUG: phalf has extra dimensions {phalf_dims}, restoring from original DS") + # Restore phalf from original dataset + if 'phalf' in DS: + DS_diurn['phalf'] = DS['phalf'].copy() + + # Now check orientation using the corrected phalf phalf_vals = DS_diurn['phalf'].values - # Check if we have at least 2 elements - if len(phalf_vals) > 1: + # Ensure we have a 1D array before checking orientation + if phalf_vals.ndim == 1 and len(phalf_vals) > 1: # Extract actual values and convert to regular Python floats first_val = float(phalf_vals[0]) last_val = float(phalf_vals[-1]) @@ -1123,6 +1156,11 @@ def main(): except Exception as e: print(f"{Yellow}Note: Could not check phalf orientation: {str(e)}") + + + + + # Create New File, set time dimension as unlimitted fullnameOUT = f'{fullnameIN[:-3]}{ext}.nc' DS_diurn.to_netcdf(fullnameOUT, unlimited_dims=model.dim_time, From 67cf6749578114871c6f6088fc91921965a62707 Mon Sep 17 00:00:00 2001 From: Courtney ML Batterson Date: Mon, 12 Jan 2026 14:06:02 -0800 Subject: [PATCH 9/9] cleaning up MarsFormat debugging text --- bin/MarsFormat.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/bin/MarsFormat.py b/bin/MarsFormat.py index f7fe95fa..08cccc29 100755 --- a/bin/MarsFormat.py +++ b/bin/MarsFormat.py @@ -1107,28 +1107,7 @@ def main(): DS_diurn[model.dim_time].attrs['long_name'] = ( f'time averaged over {nday} sols' ) - - - - - - # Safe phalf check for PCM files - # if model_type == 'pcm' and DS_diurn is not None and 'phalf' in DS_diurn: - # try: - # phalf_vals = DS_diurn['phalf'].values - # # Check if we have at least 2 elements - # if len(phalf_vals) > 1: - # # Extract actual values and convert to regular Python floats - # first_val = float(phalf_vals[0]) - # last_val = float(phalf_vals[-1]) - # if first_val > last_val: - # print(f"{Yellow}Warning: phalf orientation incorrect in diurn file, fixing...") - # DS_diurn['phalf'] = (DS_diurn['phalf'].dims, phalf_vals[::-1]) - # except Exception as e: - # print(f"{Yellow}Note: Could not check phalf orientation: {str(e)}") - - - + # Safe phalf check for PCM files # During diurn processing, phalf can gain extra dimensions from # groupby().mean() operations (similar issue to ak/bk above). @@ -1156,11 +1135,6 @@ def main(): except Exception as e: print(f"{Yellow}Note: Could not check phalf orientation: {str(e)}") - - - - - # Create New File, set time dimension as unlimitted fullnameOUT = f'{fullnameIN[:-3]}{ext}.nc' DS_diurn.to_netcdf(fullnameOUT, unlimited_dims=model.dim_time,