#930: test run ukcm2a amip (#936)

* #930: update tmp dir space

* #930: update chunking to limit filesize

* #930: handle 2+ realms in global attr

* #930: rename table to primary table

* #930: Fix missing time coord

* #930: fix missing time axis value error for clim vars

* #930: update max file size for 6hr

* #930: update daily sizing info

Author: mo-laurenboon

cdds/cdds/common/plugins/cmip7/data/model/UKCM2a-0-HH.json

@@ -29,7 +29,7 @@
     "ap6": "P3M",
     "ap7": "P3M",
     "ap8": "P1Y",
-    "ap9": "P1Y",
+    "ap9": "P6M",
     "apt": "P3M",
     "ind": "P1Y",
     "inm": "P5Y",
@@ -58,10 +58,10 @@
     "ap5": 200000,
     "apm": 200000,
     "apu": 200000,
-    "ap6": 200000,
-    "ap7": 200000,
-    "ap8": 200000,
-    "ap9": 200000,
+    "ap6": 300000,
+    "ap7": 300000,
+    "ap8": 400000,
+    "ap9": 400000,
     "apt": 200000,
     "ind": 200000,
     "inm": 200000,
@@ -83,29 +83,30 @@
     },
     "day": {
       "default": 5,
-      "19-960-1280": 1,
-      "19-961-1280": 1,
+      "19-960-1280": 0.5,
+      "19-961-1280": 0.5,
       "8-960-1280": 1,
       "8-961-1280": 1,
-      "27-960-1280": 1,
+      "27-960-1280": 0.5,
       "4-960-1280": 1,
       "5-960-1280": 1,
       "7-960-1280": 1,
-      "7-7-960-1280": 1,
+      "7-7-960-1280": 0.5,
       "4320-3604": 1,
       "40-960-1280": 1,
-      "85-960-1280": 1,
-      "85-961-1280": 1,
-      "86-960-1280": 1
+      "85-960-1280": 0.25,
+      "85-961-1280": 0.25,
+      "86-960-1280": 0.25
     },
     "6hr": {
-      "default": 1
+      "default": 1,
+      "7-960-1280": 0.5
     },
     "3hr": {
       "default": 1
     },
     "1hr": {
-      "default": 1
+      "default": 0.5
     }
   },
   "subdaily_streams": [

cdds/cdds/qc/dataset/cmip7.py

@@ -83,15 +83,15 @@ def check_filename(self, ds, filename):
             valid_filename = False
             messages.append(f"Invalid variant_label {variant_label}")
 
-        table = self.global_attributes_cache.getncattr("realm", ds)
+        primary_table = self.global_attributes_cache.getncattr("realm", ds).split(" ")[0]
         variable = filename_parts[0] + "_" + filename_parts[1]
-        if table in self._mip_tables.tables:
-            if variable not in (self._mip_tables.get_variables(table)):
+        if primary_table in self._mip_tables.tables:
+            if variable not in (self._mip_tables.get_variables(primary_table)):
                 valid_filename = False
                 messages.append(f"Invalid variable {variable} in the filename {filename}")
         else:
             valid_filename = False
-            messages.append(f"Invalid table_id Global Attribute {table} in file {filename}")
+            messages.append(f"Invalid table_id Global Attribute {primary_table} in file {filename}")
 
         if len(filename_parts) > 8 and filename_parts[8]:
             try:

cdds/cdds/qc/dataset/dataset.py

@@ -188,6 +188,10 @@ def variable_time_axis(self, var_key, atmos_timestep):
                 time_axis[filepath] = nc_file.variables["time"][:].data
                 if "time_bnds" in nc_file.variables:
                     time_bnds[filepath] = nc_file.variables["time_bnds"][:].data
+                # Time variable can have bounds in the climatology_bnds variable in CMIP7 for some data sets, add them.
+                if "climatology_bnds" in nc_file.variables:
+                    time_bnds[filepath] = nc_file.variables["climatology_bnds"][:].data
+                    self._logger.info("Adding bounds from climatology_bnds to time_bnds")
                 frequency_code = self.global_attributes_cache.getncattr("frequency", nc_file)
                 variable_id = self.global_attributes_cache.getncattr("variable_id", nc_file)
                 if frequency_code == 'subhrPt':

cdds/cdds/qc/runner.py

@@ -643,6 +643,8 @@ def get_ignored_messages(self):
             "Cell Boundaries: Bounds variable vertices_latitude and parent variable latitude have the following "
             "matching attributes ['units'].  It is recommended that only the parent variable of the bounds variable "
             "contains these attributes": "non-mandatory",
+            'Climatological Statistics: The \"time: method within years/days over years/days\" format is not correct':
+            "removed to allow production of tas and sfcWind in CMIP7",
             (
                 "the coordinate variable 'latitude' lie outside the bounding box of the "
                 "associated boundary variable 'vertices_latitude'"

mip_convert/mip_convert/new_variable.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2015-2025, Met Office.
+# (C) British Crown Copyright 2015-2026, Met Office.
 # Please see LICENSE.md for license details.
 # pylint: disable=no-member, eval-used, wildcard-import
 # pylint: disable=unused-wildcard-import
@@ -231,7 +231,7 @@ def slices_over(self, period):
                 # Slice the 'input variable(s)', since they have not yet been processed.
                 message = '-'.join([str(items) for items in date_time])
                 self.logger.debug('Creating data for "{}"'.format(message))
-                sliced_input_variables = self._slice_input_variables(date_time)
+                sliced_input_variables = self._slice_input_variables(date_time, date_times)
                 yield Variable(sliced_input_variables, self._variable_metadata)
             else:
                 # Slice the 'MIP output variable'.
@@ -844,10 +844,10 @@ def _data_dimension(self, coord, axis_direction):
             raise ValueError(message)
         return data_dimension
 
-    def _slice_input_variables(self, date_time):
+    def _slice_input_variables(self, date_time, date_times):
         input_variables = {}
-        if len(date_time) > 1 and date_time[1] != 12:
-            # don't attach New Year midnight to other months
+        if (len(date_time) > 1 and date_time[1] != 12) and date_time != date_times[-1]:
+            # don't attach New Year midnight to months that are not december or the final chunk in the slice,
             new_year_midnight = False
         else:
             new_year_midnight = True
@@ -1228,10 +1228,16 @@ def _mip_table_requested_bounds(bounds):
 
 def _setup_time_constraint(date_time, with_new_year_midnight=True):
     def time_constraint(cell):
+        """Runs only if with_new_year_midnight evaluates to True. Checks the current cell point and confirms if it
+        signifies a new year (e.g. 1980-01-01) or a new month (e.g. 1980-04-01). If so, it returns that value and
+        ensures that the data point is kept, hence maintaining time contiguity between slices."""
+        new_month = PartialDateTime(date_time[0], date_time[1] + 1, 1, 0, 0, 0, 0) if len(date_time) > 1 else ""
         return (PartialDateTime(*date_time) == cell.point or
-                PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point)
+                PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point or
+                new_month == cell.point)
 
     def time_constraint2(cell):
+        # Returns the original input cell point if with_new_year_midnight evaluates to false.
         return PartialDateTime(*date_time) == cell.point
 
     return time_constraint if with_new_year_midnight else time_constraint2