feat(systemtests): add per-test max_time override for precice-config (fix #402)

Author: PranjalManhgaye

tools/tests/systemtests.py

@@ -39,29 +39,33 @@ def main():
 
     build_args = SystemtestArguments.from_args(args.build_args)
     run_directory = Path(args.rundir)
-    if args.suites:
-        test_suites_requested = args.suites.split(',')
-        available_testsuites = TestSuites.from_yaml(
-            PRECICE_TESTS_DIR / "tests.yaml", available_tutorials)
-        test_suites_to_execute = []
-        for test_suite_requested in test_suites_requested:
-            test_suite_found = available_testsuites.get_by_name(
-                test_suite_requested)
-            if not test_suite_found:
-                logging.error(f"Did not find the testsuite with name {test_suite_requested}")
-            else:
-                test_suites_to_execute.append(test_suite_found)
-        if not test_suites_to_execute:
-            raise RuntimeError(
-                f"No matching test suites with names {test_suites_requested} found. Use print_test_suites.py to get an overview")
-        # now convert the test_suites into systemtests
-        for test_suite in test_suites_to_execute:
-            tutorials = test_suite.cases_of_tutorial.keys()
-            for tutorial in tutorials:
-                for case, reference_result in zip(
-                        test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]):
-                    systemtests_to_run.append(
-                        Systemtest(tutorial, build_args, case, reference_result))
+        if args.suites:
+            test_suites_requested = args.suites.split(',')
+            available_testsuites = TestSuites.from_yaml(
+                PRECICE_TESTS_DIR / "tests.yaml", available_tutorials)
+            test_suites_to_execute = []
+            for test_suite_requested in test_suites_requested:
+                test_suite_found = available_testsuites.get_by_name(
+                    test_suite_requested)
+                if not test_suite_found:
+                    logging.error(f"Did not find the testsuite with name {test_suite_requested}")
+                else:
+                    test_suites_to_execute.append(test_suite_found)
+            if not test_suites_to_execute:
+                raise RuntimeError(
+                    f"No matching test suites with names {test_suites_requested} found. Use print_test_suites.py to get an overview")
+            # now convert the test_suites into systemtests
+            for test_suite in test_suites_to_execute:
+                tutorials = test_suite.cases_of_tutorial.keys()
+                for tutorial in tutorials:
+                    cases = test_suite.cases_of_tutorial[tutorial]
+                    reference_results = test_suite.reference_results[tutorial]
+                    max_times = test_suite.max_times.get(tutorial, [None] * len(cases))
+                    for case, reference_result, max_time in zip(
+                            cases, reference_results, max_times):
+                        systemtest = Systemtest(tutorial, build_args, case, reference_result)
+                        systemtest.max_time = max_time
+                        systemtests_to_run.append(systemtest)
 
     if not systemtests_to_run:
         raise RuntimeError("Did not find any Systemtests to execute.")

tools/tests/systemtests/Systemtest.py

@@ -136,6 +136,7 @@ class Systemtest:
     reference_result: ReferenceResult
     params_to_use: Dict[str, str] = field(init=False)
     env: Dict[str, str] = field(init=False)
+    max_time: Optional[float] = field(default=None, init=False)
 
     def __eq__(self, other) -> bool:
         if isinstance(other, Systemtest):
@@ -321,6 +322,53 @@ def __copy_tutorial_into_directory(self, run_directory: Path):
                 file.write(ref_requested)
             self._checkout_ref_in_subfolder(PRECICE_TUTORIAL_DIR, self.tutorial.path, current_ref)
 
+    def __apply_precice_max_time_override(self):
+        """
+        If max_time is set, override the <max-time value=\"...\"/> entry in the
+        root precice-config.xml of the copied tutorial directory.
+        """
+        if self.max_time is None:
+            return
+
+        config_path = self.system_test_dir / "precice-config.xml"
+        if not config_path.exists():
+            logging.warning(
+                f\"Requested max_time override for {self}, but no precice-config.xml "
+                f\"found in {self.system_test_dir}\"
+            )
+            return
+
+        try:
+            text = config_path.read_text()
+        except Exception as e:
+            logging.warning(f\"Could not read {config_path} to apply max_time override: {e}\")
+            return
+
+        # Replace the first occurrence of value=\"...\" on a <max-time ...> tag
+        pattern = r'(<max-time[^>]*\\svalue=\")([^\"]*)(\")'
+        match = re.search(pattern, text)
+        if not match:
+            logging.warning(
+                f\"Requested max_time override for {self}, but no <max-time .../> tag "
+                f\"found in {config_path}\"
+            )
+            return
+
+        new_text = re.sub(
+            pattern,
+            rf'\\1{self.max_time}\\3',
+            text,
+            count=1,
+        )
+
+        try:
+            config_path.write_text(new_text)
+            logging.info(
+                f\"Overwrote max-time in {config_path} to {self.max_time} for {self}\"
+            )
+        except Exception as e:
+            logging.warning(f\"Failed to write updated {config_path}: {e}\")
+
     def __copy_tools(self, run_directory: Path):
         destination = run_directory / "tools"
         src = PRECICE_TOOLS_DIR
@@ -518,6 +566,7 @@ def __prepare_for_run(self, run_directory: Path):
         Prepares the run_directory with folders and datastructures needed for every systemtest execution
         """
         self.__copy_tutorial_into_directory(run_directory)
+        self.__apply_precice_max_time_override()
         self.__copy_tools(run_directory)
         self.__put_gitignore(run_directory)
         host_uid, host_gid = self.__get_uid_gid()

tools/tests/systemtests/TestSuite.py

@@ -10,6 +10,7 @@ class TestSuite:
     name: str
     cases_of_tutorial: Dict[Tutorial, List[CaseCombination]]
     reference_results: Dict[Tutorial, List[ReferenceResult]]
+    max_times: Dict[Tutorial, List[Optional[float]]] = field(default_factory=dict)
 
     def __repr__(self) -> str:
         return_string = f"Test suite: {self.name} contains:"
@@ -46,8 +47,9 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
             data = yaml.safe_load(f)
             test_suites_raw = data['test_suites']
             for test_suite_name in test_suites_raw:
-                case_combinations_of_tutorial = {}
-                reference_results_of_tutorial = {}
+                case_combinations_of_tutorial: Dict[Tutorial, List[CaseCombination]] = {}
+                reference_results_of_tutorial: Dict[Tutorial, List[ReferenceResult]] = {}
+                max_times_of_tutorial: Dict[Tutorial, List[Optional[float]]] = {}
                 # iterate over tutorials:
                 for tutorial_case in test_suites_raw[test_suite_name]['tutorials']:
                     tutorial = parsed_tutorials.get_by_path(tutorial_case['path'])
@@ -57,20 +59,34 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                     if tutorial not in case_combinations_of_tutorial:
                         case_combinations_of_tutorial[tutorial] = []
                         reference_results_of_tutorial[tutorial] = []
+                        max_times_of_tutorial[tutorial] = []
 
                     all_case_combinations = tutorial.case_combinations
                     case_combination_requested = CaseCombination.from_string_list(
                         tutorial_case['case_combination'], tutorial)
                     if case_combination_requested in all_case_combinations:
                         case_combinations_of_tutorial[tutorial].append(case_combination_requested)
-                        reference_results_of_tutorial[tutorial].append(ReferenceResult(
-                            tutorial_case['reference_result'], case_combination_requested))
+                        reference_results_of_tutorial[tutorial].append(
+                            ReferenceResult(
+                                tutorial_case['reference_result'],
+                                case_combination_requested,
+                            )
+                        )
+                        max_times_of_tutorial[tutorial].append(
+                            tutorial_case.get('max_time')
+                        )
                     else:
                         raise Exception(
                             f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}")
 
-                testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial,
-                                            reference_results_of_tutorial))
+                testsuites.append(
+                    TestSuite(
+                        test_suite_name,
+                        case_combinations_of_tutorial,
+                        reference_results_of_tutorial,
+                        max_times_of_tutorial,
+                    )
+                )
 
         return cls(testsuites)