Changes to enable easily turning on CUPiD diagnostics from your CTSM case

.gitmodules

@@ -121,6 +121,18 @@ fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial
 
+[submodule "cupid"]
+# CUPiD does model post-processing
+path = tools/CUPiD
+#url = https://github.com/NCAR/CUPiD.git
+url = https://github.com/wwieder/CUPiD.git
+#fxtag = v0.6.0
+#branch = land_only
+fxtag = dd65b0be6d70a3158549d636ec26376f94c27d40
+fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
+fxDONOTUSEurl = https://github.com/NCAR/CUPiD.git
+
 [submodule "doc-builder"]
 path = doc/doc-builder
 url = https://github.com/ESMCI/doc-builder

README

@@ -80,6 +80,7 @@ README ------------------- This file
 README.md ---------------- File that displays on github under https::/github.com/ESCOMP/CTSM.git
 README.rst --------------- File that displays under the project in github
 README_GITFLEXIMOD.rst --- Information on how to work with git-fleximod for CTSM
+README_DIAGNOSTICS.md ---- Quickstart on using CUPiD to plot diagnostics for your CTSM case
 WhatsNewInCTSM5.4.md ----- Overview document of the changes between ctsm5.3 and ctsm5.4
 Copyright ---------------- CESM Copyright file
 doc/UpdateChangeLog.pl --- Script to add documentation on a tag to the

README.DIAGNOSTICS

@@ -0,0 +1,61 @@
+# QuickStart to Running Diagnostics of your Case with CUPiD (CESM Unified Postprocessing and Diagnostics)
+
+[!NOTE]
+Land Diagnostics Cheat Sheet is here:
+
+This document has more details on more options and such
+
+https://ncar.github.io/CUPiD/run_cesm.html
+
+## Initial setup steps to do first time
+
+This step is something you need to do the first time you want to run CUPiD for a case.
+And you only need to do it again if the CUPiD environment changes -- or if you need to
+setup the environment for a different CTSM checkout.
+
+``` shell
+# CUPiD instructions for this is here: https://ncar.github.io/CUPiD/index.html#installing
+cd tools/CUPiD
+mamba env create --yes -f environments/cupid-infrastructure.yml
+mamba env create --yes -f environments/cupid-analysis.yml
+# Check that the environment is valid
+conda activate cupid-infrastructure
+which cupid-diagnostics
+# Should return something like:
+# $HOME/conda-envs/cupid-infrastructure/bin/cupid-diagnostics
+# If it returns "Command not found." something is wrong with the environment
+
+# Instructions for the next part are here:
+# https://ncar.github.io/CUPiD/index.html#note
+# Now make sure the conda environments can be used by Jupyter notebooks
+conda activate cupid-analysis
+python -m ipykernel install --user --name=cupid-analysis
+conda activate cupid-infrastructure
+python -m ipykernel install --user --name=cupid-infrastructure
+```
+
+<!-- ========================================================================================== -->
+#### IMPORTANT NOTE: You have to REMAKE THE CUPiD ENVIRONMENTS for a different CTSM Checkout
+<!-- ========================================================================================== -->
+
+
+> [!IMPORTANT]
+> If you want to use CUPiD in a different clone of CTSM -- you'll need to remake the environments there.
+
+## Create your case using the CUPiD user-mod
+
+This is similar to setting up any case, such as documented in the README and Quickstart guides.
+
+``` shell
+./create_newcase --case testIwCUPiD --res f09_t232 --compset I2000Clm60BgcCrop --user-mods-dirs clm-CUPiD
+```
+
+## Run CUPID in your case
+
+After st_archive has run do the following. With RUN_POSTPROCESSING set to TRUE this will happen
+with each case submission automatically. But, if you want to run it separately...
+
+
+``` shell
+./case.submit --only-job case.cupid
+```

cime_config/SystemTests/ctsm_test_status.py

@@ -0,0 +1,30 @@
+
+from CIME.test_status import *
+
+CUPID_PHASE = "CUPID"
+
+ALL_PHASES = [
+    CREATE_NEWCASE_PHASE,
+    XML_PHASE,
+    SETUP_PHASE,
+    NAMELIST_PHASE,
+    SHAREDLIB_BUILD_PHASE,
+    MODEL_BUILD_PHASE,
+    SUBMIT_PHASE,
+    RUN_PHASE,
+    COMPARE_PHASE,
+    BASELINE_PHASE,
+    THROUGHPUT_PHASE,
+    MEMCOMP_PHASE,
+    MEMLEAK_PHASE,
+    STARCHIVE_PHASE,
+    CUPID_PHASE,
+    GENERATE_PHASE,
+]
+
+# Extend the TestStatus class to include the CUPID phase and have CUPID phase within ALL_PHASES
+class CTSMTestStatus(TestStatus):
+
+    def __init__(self, case):
+        super().__init__(case, ALL_PHASES)
+        self._test_status = CTSMTestStatus(test_dir=case.get_value("CASEROOT"), test_name=self.case.get_value("CASEBASEID") )

cime_config/SystemTests/smscupid.py

@@ -0,0 +1,82 @@
+from CIME.SystemTests.sms import SMS
+from ctsm_test_status import *
+
+
+class SMSCUPID(SMS):
+    def __init__(self, case):
+        super().__init__(case)
+        self._test_status = CTSMTestStatus(
+            test_dir=self._case.get_value("CASEROOT"),
+            test_name=self._case.get_value("CASEBASEID"),
+        )
+
+    def setup_indv(
+        self, clean=False, test_mode=False, reset=False, keep=False, disable_git=False
+    ):
+        """
+        Perform an individual setup
+        """
+        super().setup_indv(
+            clean=clean,
+            test_mode=test_mode,
+            reset=reset,
+            keep=keep,
+            disable_git=disable_git,
+        )
+        # Make sure the st_archiver is turned on, as CUPiD only runs after it runs
+        self._case.set_value("DOUT_S", True)
+
+    def run_indv(
+        self,
+        suffix="base",
+        st_archive=True,
+        cupid=True,
+        submit_resubmits=None,
+        keep_init_generated_files=False,
+    ):
+        """
+        Perform an individual run. Raises an EXCEPTION on fail.
+
+        Just add the CUPiD phase after the standard run.
+        """
+        super().run_indv(
+            suffix=suffix,
+            st_archive=st_archive,
+            submit_resubmits=submit_resubmits,
+            keep_init_generated_files=keep_init_generated_files,
+        )
+        self._phase_modifying_call(CUPID_PHASE, self._cupid_case_test)
+
+    def case_test_cupid(self, testdir="cupid_test"):
+        # create the run directory testdir
+        if os.path.exists(testdir):
+            logger.info("Removing existing test directory {}".format(testdir))
+            shutil.rmtree(testdir)
+        # Check that the CUPid postprocessing directories and config file exist
+        cupid_dir = os.path.join(self._case.get_value("CASEROOT"), "cupid-postprocessing" )
+        notebooks_dir = os.path.join(cupid_dir, "compute_notebooks")
+        data_dir = os.path.join(cupid_dir, "temp_data")
+        for dir in [cupid_dir, notebooks_dir, data_dir]:
+           expect( os.isdir(dir),
+                   "CUPiD postprocessing directory {} does not exist".format(dir) )
+        cupid_config = os.path.join(cupid_dir, "config.yml")
+        expect( os.isfile(cupid_config),
+                "CUPiD config file {} does not exist".format(cupid_config) )
+        # TODO: Add more checks about more files that should exist
+
+
+        # TODO: Populate the testdir with data files, config files and notebooks from the cupid-postprocessing directory
+
+        # TODO: Save various files to the baseline directory to use for BASELINE comparison
+
+        return True
+
+    def _cupid_case_test(self):
+        # For the st_archiver this test is under the case object
+        # Here we create it in this object, but probably should be moved to under the case object in CIME
+        result = self.test_cupid()
+        with self._test_status:
+            if result:
+                self._test_status.set_status(CUPID_PHASE, TEST_PASS_STATUS)
+            else:
+                self._test_status.set_status(CUPID_PHASE, TEST_FAIL_STATUS)

cime_config/SystemTests/smspostproc.py

@@ -0,0 +1,19 @@
+from CIME.SystemTests.sms import SMS
+
+class SMSPOSTPROC(SMS):
+    def setup_indv(
+        self, clean=False, test_mode=False, reset=False, keep=False, disable_git=False
+    ):
+        """
+        Perform an individual setup
+        """
+        super().setup_indv(
+            clean=clean,
+            test_mode=test_mode,
+            reset=reset,
+            keep=keep,
+            disable_git=disable_git,
+        )
+        # Make sure the st_archiver is turned on, as CUPiD only runs after it runs
+        self._case.set_value("DOUT_S", True)
+        self._case.set_value("RUN_POSTPROCESSING", True)

cime_config/config_tests.xml

@@ -185,6 +185,33 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="SMSPOSTPROC">
+    <DESC>Run a smoke startup test with st_archive and post-processing on</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>TRUE</DOUT_S>
+    <RUN_POSTPROCESSING>TRUE</RUN_POSTPROCESSING>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <STOP_OPTION>nyears</STOP_OPTION>
+    <STOP_N>5</STOP_N>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
+  <test NAME="SMSCUPID">
+    <DESC>Run a smoke startup test with CUPiD on</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>TRUE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <RUN_POSTPROCESSING>TRUE</RUN_POSTPROCESSING>
+    <!-- TODO: Add CUPID Settings -->
+    <STOP_OPTION>nyears</STOP_OPTION>
+    <STOP_N>5</STOP_N>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
 <!--
 SSP    smoke CLM spinup test (only valid for CLM compsets with CLM45)
        do an initial spin test (setting CLM_ACCELERATED_SPINUP to on)

cime_config/testdefs/testlist_clm.xml

@@ -17,6 +17,7 @@
         decomp_init: Initialization tests specifically for examining the PE layout decomposition initialization
         uhr_decomp_init: Initialization tests at Ultra High Resolution - specifically for examining the PE layout decomposition initialization
         interim_restart: Tests having to do with interim restart capability.
+        aux_cupic: Tests that run the CUPiD postprocessing workflow
 -->
 <testlist version="2.0">
   <test name="ERI_D_Ld9" grid="f10_f10_mg37" compset="I1850Clm60Bgc" testmods="clm/default">
@@ -100,6 +101,71 @@
       <option name="comment">Repeat no anthropogenic compset with t232 and Crujra while the defaults have not changed</option>
     </options>
   </test>
+  <test name="SMS_POSTPROC_Ly5" grid="f45_f45_mg37" compset="IHistClm60BgcCrujra" testmods="clm/CUPiD">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="aux_cupid"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:40:00</option>
+      <option name="comment">Simple smoke test running st_archive with CUPiD on for LND, requires cupid conda environment</option>
+    </options>
+  </test>
+  <test name="SMS_POSTPROC_Ly5" grid="f45_f45_mg37" compset="IHistClm60BgcCrujra" testmods="clm/CUPiD_web">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_cupid"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:40:00</option>
+      <option name="comment">Simple smoke test running st_archive with CUPiD on for LND and create web-pages, requires cupid conda environment</option>
+    </options>
+  </test>
+  <test name="SMS_POSTPROC_Ly5" grid="f45_f45_mg37" compset="IHistClm60BgcCrujra" testmods="clm/CUPiD_LDF">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_cupid"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:40:00</option>
+      <option name="comment">Simple smoke test running st_archive with CUPiD on for the LDF, requires cupid conda environment</option>
+    </options>
+  </test>
+  <test name="SMS_POSTPROC_Ly5" grid="f45_f45_mg37" compset="I2000Clm60BgcCrujra" testmods="clm/CUPiD_LDF">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_cupid"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:40:00</option>
+      <option name="comment">Simple smoke test for a I2000 case running st_archive with CUPiD on for the LDF, requires cupid conda environment</option>
+    </options>
+  </test>
+  <test name="SMS_POSTPROC_Ly5" grid="f45_f45_mg37" compset="I2000Clm60SpCrujra" testmods="clm/CUPiD_LDF">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_cupid"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:40:00</option>
+      <option name="comment">Simple smoke test for a I2000 SP case running st_archive with CUPiD on for the LDF, requires cupid conda environment</option>
+    </options>
+  </test>
+  <test name="SMS_POSTPROC_Ly5" grid="f45_f45_mg37" compset="IHistClm60BgcCrujra" testmods="clm/CUPiD_ILAMB">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_cupid"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:40:00</option>
+      <option name="comment">Simple smoke test running st_archive with CUPiD on for the ILAMB, requires cupid conda environment</option>
+    </options>
+  </test>
+  <test name="SMS_POSTPROC_PS_Ly2" grid="f09_f09_mt232" compset="IHistClm60BgcCrujra" testmods="clm/CUPiD_ROF">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_cupid"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:40:00</option>
+      <option name="comment">Simple smoke test running st_archive with CUPiD on for the ROF model requires cupid conda environment</option>
+      <option name="comment">At f09 as that is the resolution supported for MOSART</option>
+    </options>
+  </test>
   <test name="SMS_D_Ld5_PS" grid="f09_f09_mt232" compset="I1850Clm60BgcCrop" testmods="clm/f09_ObscureStreamOpts">
     <machines>
       <machine name="derecho" compiler="gnu" category="aux_clm"/>

cime_config/testdefs/testmods_dirs/clm/CUPiD_ILAMB/include_user_mods

@@ -0,0 +1 @@
+../../../../usermods_dirs/clm/CUPiD

cime_config/testdefs/testmods_dirs/clm/CUPiD_ILAMB/shell_commands

@@ -0,0 +1,3 @@
+#!/bin/bash
+./xmlchange CUPID_RUN_ILAMB=TRUE
+

cime_config/testdefs/testmods_dirs/clm/CUPiD_LDF/include_user_mods

@@ -0,0 +1 @@
+../../../../usermods_dirs/clm/CUPiD

cime_config/testdefs/testmods_dirs/clm/CUPiD_LDF/shell_commands

@@ -0,0 +1,8 @@
+#!/bin/bash
+./xmlchange CUPID_RUN_LDF=TRUE
+
+# Base case setttings to match the case just run
+./xmlchange CUPID_BASE_STARTDATE='$RUN_STARTDATE'
+./xmlchange CUPID_BASE_CLIMO_END_YEAR=2005
+./xmlchange CUPID_BASE_CLIMO_N_YEAR='$STOP_N'
+./xmlchange CUPID_BASE_STOP_N='$STOP_N'

cime_config/testdefs/testmods_dirs/clm/CUPiD_ROF/include_user_mods

@@ -0,0 +1 @@
+../../../../usermods_dirs/clm/CUPiD

cime_config/testdefs/testmods_dirs/clm/CUPiD_ROF/shell_commands

@@ -0,0 +1,4 @@
+#!/bin/bash
+./xmlchange CUPID_RUN_ROF=TRUE
+./xmlchange CUPID_RUN_LND=FALSE
+

cime_config/testdefs/testmods_dirs/clm/CUPiD_web/include_user_mods

@@ -0,0 +1 @@
+../../../../usermods_dirs/clm/CUPiD

cime_config/testdefs/testmods_dirs/clm/CUPiD_web/shell_commands

@@ -0,0 +1,3 @@
+#!/bin/bash
+./xmlchange CUPID_GEN_HTML=TRUE
+

cime_config/usermods_dirs/clm/CUPiD/README

@@ -0,0 +1,4 @@
+This usermod turns on running CUPiD to get plots and diagnostics as part of a case. It sets the most common settings
+for a CTSM case.
+
+To run with CUPiD you'll need to activate the cupid-infrastructure conda environment

cime_config/usermods_dirs/clm/CUPiD/include_user_mods

@@ -0,0 +1 @@
+../_includes/hist_2000-2005