add summarize option

Author: robertjwilson

oceanval/data/variable_summary_template.ipynb

@@ -117,7 +117,7 @@
    },
    "outputs": [],
    "source": [
-    "data_dir = \"/data/proteus1/scratch/rwi/oceanval_testing/oceanval_summaries/data\"\n"
+    "data_dir = \"../../../oceanval_summaries/data\""
    ]
   },
   {
@@ -132,7 +132,12 @@
    "outputs": [],
    "source": [
     "data_paths = nc.glob(f\"{data_dir}/{variable}/\")\n",
-    "i_fig = 1"
+    "i_fig = 1\n",
+    "ff = f\"{data_dir}/{variable}/summaries_config.pkl\"\n",
+    "with open(ff, \"rb\") as f:\n",
+    "    summaries = pickle.load(f)\n",
+    "clim_period = f\"{summaries[variable].climatology_years[0]}-{summaries[variable].climatology_years[1]}\"\n",
+    "robust = summaries[variable].robust"
    ]
   },
   {
@@ -150,7 +155,7 @@
     "if len(surface_clim) > 0:\n",
     "    md(f\"## Sea surface climatology of {variable}\") \n",
     "    ds = nc.open_data(surface_clim[0], checks = False)  \n",
-    "    ds.pub_plot(robust = True)\n"
+    "    ds.pub_plot(robust = robust)\n"
    ]
   },
   {
@@ -165,7 +170,7 @@
    "outputs": [],
    "source": [
     "if len(surface_clim) > 0:\n",
-    "    md(f\"**Figure {i_fig}**: Sea surface climatology of {variable}.\")\n",
+    "    md(f\"**Figure {i_fig}**: Sea surface climatology of {variable} ({clim_period}).\")\n",
     "    i_fig += 1"
    ]
   },
@@ -184,7 +189,7 @@
     "if len(verticalmean_clim) > 0:\n",
     "    md(f\"## Vertical mean climatology of {variable}\") \n",
     "    ds = nc.open_data(verticalmean_clim[0], checks = False)  \n",
-    "    ds.pub_plot(robust = True)"
+    "    ds.pub_plot(robust = robust)"
    ]
   },
   {
@@ -198,8 +203,9 @@
    },
    "outputs": [],
    "source": [
-    "md(f\"**Figure {i_fig}**: Climatoloy of vertical average {variable}.\")\n",
-    "i_fig += 1"
+    "if len(verticalmean_clim) > 0:\n",
+    "    md(f\"**Figure {i_fig}**: Climatology of vertical average {variable} ({clim_period}).\")\n",
+    "    i_fig += 1"
    ]
   },
   {
@@ -215,9 +221,9 @@
    "source": [
     "verticalintegrated_clim = [x for x in data_paths if \"verticalintegrated_climatology\" in x]\n",
     "if len(verticalintegrated_clim) > 0:\n",
-    "    md(f\"## Vertical integrated climatology of {variable}\") \n",
+    "    md(f\"## Vertical integrated climatology of {variable}\")\n",
     "    ds = nc.open_data(verticalintegrated_clim[0], checks = False)  \n",
-    "    ds.pub_plot(robust = True)\n",
+    "    ds.pub_plot(robust = robust)\n",
     "    "
    ]
   },
@@ -233,7 +239,7 @@
    "outputs": [],
    "source": [
     "if len(verticalintegrated_clim) > 0:\n",
-    "    md(f\"**Figure {i_fig}**: Climatology of vertical integrated {variable}.\")\n",
+    "    md(f\"**Figure {i_fig}**: Climatology of vertical integrated {variable} ({clim_period}).\")\n",
     "    i_fig += 1"
    ]
   },

oceanval/parsers.py

@@ -922,10 +922,11 @@ def add_summary(
         model_variable=None,
         long_name=None,
         short_name=None,
-        trends = {"period":[-1E9, 1E9], "window":1},
+        trends = None, 
         vertical_integration=False,
         vertical_mean =False,
-        climatology_years = None
+        climatology_years = None,
+        robust = False
     ):
         """
         Add a variable to be summarized from model output.
@@ -948,6 +949,8 @@ def add_summary(
             Whether to vertically integrate the variable.
         vertical_mean : bool, default False
             Whether to vertically average the variable.
+        robust : bool, default False
+            Whether to use a robust plot for climatologies 
         
         Returns
         -------
@@ -990,6 +993,9 @@ def add_summary(
             except (TypeError, ValueError) as e:
                 raise ValueError("climatology_years values must be integers")
 
+        # check robust is boolean
+        if not isinstance(robust, bool):
+            raise ValueError("robust must be a boolean value")
 
         # Validate required parameters
         if name is None:
@@ -1003,10 +1009,11 @@ def add_summary(
             raise ValueError("Model variable must be provided")
 
         # check trends is a dict with period and window
-        if not isinstance(trends, dict):
-            raise ValueError("trends must be a dictionary with keys 'period' and 'window'")
-        if "period" not in trends or "window" not in trends:
-            raise ValueError("trends dictionary must contain 'period' and 'window' keys")
+        if trends is not None:
+            if not isinstance(trends, dict):
+                raise ValueError("trends must be a dictionary with keys 'period' and 'window'")
+            if "period" not in trends or "window" not in trends:
+                raise ValueError("trends dictionary must contain 'period' and 'window' keys")
 
         # Create variable if it doesn't exist, or get existing one
         if getattr(self, name, None) is None:
@@ -1016,6 +1023,7 @@ def add_summary(
             var = self[name]
 
         var.trends = trends
+        var.robust = robust 
         # Set attributes
         var.name = name
         var.start = start

oceanval/summarize.py

@@ -619,93 +619,94 @@ def summarize(
             continue
 
         # Open and process the data
-        try:
-            ds = nc.open_data(all_files, checks=False)
+        ds = nc.open_data(all_files, checks=False)
 
-            # Check if variable exists
-            if model_var not in ds.variables:
-                print(f"  Warning: Variable {model_var} not found in files")
-                continue
+        # Check if variable exists
+        if model_var not in ds.variables:
+            print(f"  Warning: Variable {model_var} not found in files")
+            continue
 
-            # Subset to variable of interest
-            ds.subset(variables=model_var)
+        # Subset to variable of interest
+        ds.subset(variables=model_var)
 
-            # Apply time range
+        # Apply time range
 
-            # Apply spatial subsetting
-            if lon_lim is not None and lat_lim is not None:
-                ds.subset(lon=lon_lim, lat=lat_lim)
+        # Apply spatial subsetting
+        if lon_lim is not None and lat_lim is not None:
+            ds.subset(lon=lon_lim, lat=lat_lim)
 
-            # Handle missing values
-            if as_missing is not None:
-                ds.as_missing(as_missing)
+        # Handle missing values
+        if as_missing is not None:
+            ds.as_missing(as_missing)
 
-            # Apply depth range if specified
-            ds.merge("time")
-            ds.tmean("year")
-            ds.run()
-
-            # now do the climatology
-            clim_years = summaries[var_name].climatology_years
-            ds_clim = ds.copy()
-            ds_clim.subset(years=range(clim_years[0], clim_years[1] + 1))
-            ds_clim.top()
-            ds_clim.tmean()
+        # Apply depth range if specified
+        ds.merge("time")
+        ds.tmean("year")
+        ds.run()
+
+        # now do the climatology
+        clim_years = summaries[var_name].climatology_years
+        ds_clim = ds.copy()
+        ds_clim.subset(years=range(clim_years[0], clim_years[1] + 1))
+        ds_clim.top()
+        ds_clim.tmean()
 
-            # Prepare output filename
-            out_file = f"{summary_dir}/data/{var_name}/{var_name}_surface_climatology.nc"
-            os.makedirs(os.path.dirname(out_file), exist_ok=True)
+        # Prepare output filename
+        out_file = f"{summary_dir}/data/{var_name}/{var_name}_surface_climatology.nc"
+        os.makedirs(os.path.dirname(out_file), exist_ok=True)
 
-            # Check if we should overwrite
-            if os.path.exists(out_file) and not overwrite:
-                print(f"  File exists and overwrite=False, skipping: {out_file}")
-                continue
+        # Check if we should overwrite
+        if os.path.exists(out_file) and not overwrite:
+            print(f"  File exists and overwrite=False, skipping: {out_file}")
+            continue
 
-            # Save the result
-            # delete if it iexists
+        # Save the result
+        # delete if it iexists
+        if os.path.exists(out_file):
+            os.remove(out_file)
+        ds_clim.to_nc(out_file, overwrite=True, zip=True)
+        print(f"  Saved: {out_file}")
+        trend_info = summaries[var_name].trends
+        trends = trend_info is not None
+        # now figure out if vertical mean is neeed
+        if summaries[var_name].vertical_mean:
+            ds_vertmean = ds.copy()
+            ds_vertmean.subset(years=clim_years)
+            ds_vertmean.tmean()
+            thickness = session_info["ds_thickness"]
+            if thickness is None:
+                ds_vertmean.vertical_mean(fixed = True)
+            else:
+                ds_vertmean.vertical_mean(thickness = ds_cell_thickness)
+            # climatological years
+            out_file = f"{summary_dir}/data/{var_name}/{var_name}_verticalmean_climatology.nc"
+            os.makedirs(os.path.dirname(out_file), exist_ok=True)
             if os.path.exists(out_file):
                 os.remove(out_file)
-            ds_clim.to_nc(out_file, overwrite=True, zip=True)
+            ds_vertmean.to_nc(out_file, overwrite=True, zip=True)
             print(f"  Saved: {out_file}")
-            # now figure out if vertical mean is neeed
-            if summaries[var_name].vertical_mean:
-                ds_vertmean = ds.copy()
-                ds_vertmean.subset(years=clim_years)
-                ds_vertmean.tmean()
-                thickness = session_info["ds_thickness"]
-                if thickness is None:
-                    ds_vertmean.vertical_mean(fixed = True)
-                else:
-                    ds_vertmean.vertical_mean(thickness = ds_cell_thickness)
-                # climatological years
-                out_file = f"{summary_dir}/data/{var_name}/{var_name}_verticalmean_climatology.nc"
-                os.makedirs(os.path.dirname(out_file), exist_ok=True)
-                if os.path.exists(out_file):
-                    os.remove(out_file)
-                ds_vertmean.to_nc(out_file, overwrite=True, zip=True)
-                print(f"  Saved: {out_file}")
 
-            # now do vertical integration if needed
-            if summaries[var_name].vertical_integration:
-                ds_vertint = ds.copy()
-                ds_vertint.subset(years=clim_years)
-                ds_vertint.tmean()
-                thickness = session_info["ds_thickness"]
-                if thickness is None:
-                    ds_vertint.vertical_integration(fixed = True)
-                else:
-                    ds_vertint.vertical_integration(thickness = ds_cell_thickness)
-                # climatological years
-                out_file = f"{summary_dir}/data/{var_name}/{var_name}_verticalintegrated_climatology.nc"
-                os.makedirs(os.path.dirname(out_file), exist_ok=True)
-                if os.path.exists(out_file):
-                    os.remove(out_file)
-                ds_vertint.to_nc(out_file, overwrite=True, zip=True)
-                print(f"  Saved: {out_file}")
+        # now do vertical integration if needed
+        if summaries[var_name].vertical_integration:
+            ds_vertint = ds.copy()
+            ds_vertint.subset(years=clim_years)
+            ds_vertint.tmean()
+            thickness = session_info["ds_thickness"]
+            if thickness is None:
+                ds_vertint.vertical_integration(fixed = True)
+            else:
+                ds_vertint.vertical_integration(thickness = ds_cell_thickness)
+            # climatological years
+            out_file = f"{summary_dir}/data/{var_name}/{var_name}_verticalintegrated_climatology.nc"
+            os.makedirs(os.path.dirname(out_file), exist_ok=True)
+            if os.path.exists(out_file):
+                os.remove(out_file)
+            ds_vertint.to_nc(out_file, overwrite=True, zip=True)
+            print(f"  Saved: {out_file}")
 
-                # now do the spatial mean timeseries
+            # now do the spatial mean timeseries
+            if trends:
                 ds_trends = ds.copy()
-                trend_info = summaries[var_name].trends
                 period = trend_info["period"]
                 ds_trends.subset(years = range(period[0], period[1] + 1))
                 window = trend_info["window"]
@@ -715,7 +716,6 @@ def summarize(
                     ds_trends.vertical_integration(fixed = True)
                 else:
                     ds_trends.vertical_integration(thickness = ds_cell_thickness)
-                ds_trends.fix_amm7_grid()
                 ds_trends.spatial_sum(by_area = True)
                 out_file = f"{summary_dir}/data/{var_name}/{var_name}_verticalintegrated_spatialsumtimeseries.nc"
                 os.makedirs(os.path.dirname(out_file), exist_ok = True)
@@ -724,6 +724,7 @@ def summarize(
 
                 ds_trends.to_nc(out_file, overwrite=True, zip=True)
 
+        if trends:
             ds_trends = ds.copy()
             ds_trends.top()
             ds_trends.spatial_mean()
@@ -743,10 +744,6 @@ def summarize(
             ds_trends.to_nc(out_file, overwrite=True, zip=True)
 
 
-        except Exception as e:
-            print(f"  Error processing {var_name}: {str(e)}")
-            warnings.warn(f"Failed to process {var_name}: {str(e)}")
-            continue
 
         # Save summaries configuration
         config_file = os.path.join(f"{summary_dir}/data/{var_name}", "summaries_config.pkl")