Merge pull request #4952 from ESMCI/jgfouca/imprv_baseline_handling

Improve baseline handling when regenerating baselines

Author: jgfouca

CIME/case/case_cmpgen_namelists.py

@@ -95,11 +95,14 @@ def _do_full_nl_gen_impl(case, test, generate_name, baseline_root=None):
 
     shutil.copytree(casedoc_dir, baseline_casedocs)
 
-    for item in glob.glob(os.path.join(test_dir, "user_nl*")):
-        preexisting_baseline = os.path.join(baseline_dir, os.path.basename(item))
-        if os.path.exists(preexisting_baseline):
-            os.remove(preexisting_baseline)
+    # Note: If it is ever the case that nml cmp/gen affects more files than
+    # just user_nl* and CaseDocs, this will break assumptions all across CIME.
+
+    # Remove any previous user_nl files
+    for item in glob.glob(os.path.join(baseline_dir, "user_nl*")):
+        os.remove(item)
 
+    for item in glob.glob(os.path.join(test_dir, "user_nl*")):
         safe_copy(item, baseline_dir, preserve_meta=False)
 
 

CIME/hist_utils.py

@@ -5,6 +5,7 @@
 import os
 import re
 import filecmp
+import shutil
 
 from CIME.XML.standard_module_setup import *
 from CIME.config import Config
@@ -651,6 +652,20 @@ def _generate_baseline_impl(case, baseline_dir=None, allow_baseline_overwrite=Fa
     ):
         expect(False, " Cowardly refusing to overwrite existing baseline directory")
 
+    # Remove stale baseline files from a previous run so they don't linger when the
+    # new run no longer produces them.  CaseDocs and user_nl* are managed by the
+    # namelist-generation phase (which already ran), so we preserve those.
+    preserve_list = ["CaseDocs", BLESS_LOG_NAME, TEST_STATUS_FILENAME]
+    if os.path.isdir(basegen_dir):
+        for item in os.listdir(basegen_dir):
+            if item in preserve_list or item.startswith("user_nl"):
+                continue
+            item_path = os.path.join(basegen_dir, item)
+            if os.path.isfile(item_path):
+                os.remove(item_path)
+            elif os.path.isdir(item_path):
+                shutil.rmtree(item_path)
+
     comments = "Generating baselines into '{}'\n".format(basegen_dir)
     num_gen = 0
     for model in _iter_model_file_substrs(case):

CIME/test_scheduler.py

@@ -28,7 +28,6 @@
     get_project,
     get_timestamp,
     get_cime_default_driver,
-    clear_folder,
     CIMEError,
 )
 from CIME.config import Config
@@ -360,10 +359,7 @@ def __init__(
                     if os.path.isdir(test_baseline):
                         existing_baselines.append(test_baseline)
                         if allow_baseline_overwrite and run_count == 0:
-                            if self._namelists_only:
-                                clear_folder(os.path.join(test_baseline, "CaseDocs"))
-                            else:
-                                clear_folder(test_baseline)
+                            pass
                         elif skip_tests_with_existing_baselines:
                             tests_to_skip.append(test_name)
                 expect(

CIME/tests/test_sys_test_scheduler.py

@@ -586,6 +586,86 @@ def test_e_test_inferred_compiler(self):
         )
         self.assertEqual(os.path.split(result)[1], "gnuX")
 
+    def test_f_baseline_not_cleared_on_failure(self):
+        """
+        Verify that existing baselines are preserved when a regeneration attempt
+        fails.  Before the fix, clear_folder ran in the TestScheduler constructor
+        (before the test executed), so a failing run left the case with no
+        baselines.  After the fix, clearing is deferred to _generate_baseline_impl
+        which is only reached when the run phase succeeds.
+
+        Steps:
+          1. Generate baselines with TESTRUNFAIL_PASS set (test passes).
+          2. Attempt regeneration WITHOUT TESTRUNFAIL_PASS (run fails).
+             Baselines must remain intact.
+          3. Compare with TESTRUNFAIL_PASS set – should succeed because the
+             original baselines from step 1 are still present.
+        """
+        test_name = "TESTRUNFAIL_P1.f19_g16.A"
+        if self._config.create_test_flag_mode == "e3sm":
+            genargs = ["-g", "-o", "-b", self._baseline_name, test_name]
+            compargs = ["-c", "-b", self._baseline_name, test_name]
+        else:
+            genargs = ["-g", self._baseline_name, "-o", test_name]
+            compargs = ["-c", self._baseline_name, test_name]
+
+        # Step 1: Generate initial baselines with a passing run.
+        os.environ["TESTRUNFAIL_PASS"] = "True"
+        try:
+            self._create_test(genargs)
+        finally:
+            del os.environ["TESTRUNFAIL_PASS"]
+
+        baseline_dir = os.path.join(self._baseline_area, self._baseline_name)
+        self.assertTrue(
+            os.path.isdir(baseline_dir), "Baseline dir should exist after generation"
+        )
+        baseline_cases = glob.glob(os.path.join(baseline_dir, "*TESTRUNFAIL*"))
+        self.assertEqual(
+            len(baseline_cases), 1, "Expected exactly one TESTRUNFAIL baseline dir"
+        )
+
+        # Collect the set of regular files created by the baseline generation.
+        # CaseDocs is a directory managed separately by the namelist phase and
+        # may be legitimately refreshed during a subsequent SETUP, so we only
+        # track files here.
+        baseline_case_dir = baseline_cases[0]
+        baseline_files_before = {
+            f
+            for f in os.listdir(baseline_case_dir)
+            if os.path.isfile(os.path.join(baseline_case_dir, f))
+        }
+        self.assertGreater(
+            len(baseline_files_before),
+            0,
+            "Baseline must contain at least one file after generation",
+        )
+
+        # Step 2: Attempt regeneration WITHOUT TESTRUNFAIL_PASS – run will fail.
+        # Our fix ensures that baseline files are NOT cleared before the run
+        # completes successfully.
+        self._create_test(genargs, run_errors=True)
+
+        baseline_cases_after = glob.glob(os.path.join(baseline_dir, "*TESTRUNFAIL*"))
+        self.assertEqual(len(baseline_cases_after), 1)
+        baseline_files_after = {
+            f
+            for f in os.listdir(baseline_cases_after[0])
+            if os.path.isfile(os.path.join(baseline_cases_after[0], f))
+        }
+        self.assertEqual(
+            baseline_files_before,
+            baseline_files_after,
+            msg="Baseline files were modified by a failed generation attempt",
+        )
+
+        # Step 3: Comparison against the preserved baselines should succeed.
+        os.environ["TESTRUNFAIL_PASS"] = "True"
+        try:
+            self._create_test(compargs)
+        finally:
+            del os.environ["TESTRUNFAIL_PASS"]
+
 
 if __name__ == "__main__":
     unittest.main()