systemtests: add per-test max_time override for precice-config

Author: AdityaGupta716

changelog-entries/402.md

@@ -0,0 +1 @@
+- Add optional `max_time` and `max_time_windows` fields to `tests.yaml` entries to override `<max-time>` or `<max-time-windows>` in `precice-config.xml` per test case.

tools/tests/README.md

@@ -118,6 +118,8 @@ In order for the systemtests to pick up the tutorial we need to define a `metada
 To add a testsuite just open the `tests.yaml` file and use the output of `python print_case_combinations.py` to add the right case combinations you want to test. Note that you can specify a `reference_result` which is not yet present. The `generate_reference_data.py` will pick that up and create it for you.
 Note that its important to carefully check the paths of the `reference_result` in order to not have typos in there. Also note that same cases in different testsuites should use the same `reference_result`.
 
+To cap the preCICE simulation time for a specific test without editing `precice-config.xml`, add an optional `max_time` (positive float, overrides `<max-time>`) or `max_time_windows` (positive integer, overrides `<max-time-windows>`) field to the tutorial entry. Applies to both test runs and reference result generation.
+
 ### Generate reference results
 
 Since we need data to compare against, you need to run `python generate_reference_data.py`. This process might take a while.

tools/tests/generate_reference_results.py

@@ -108,10 +108,14 @@ def main():
     for test_suite in test_suites:
         tutorials = test_suite.cases_of_tutorial.keys()
         for tutorial in tutorials:
-            for case, reference_result in zip(
-                    test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]):
+            max_times = test_suite.max_times.get(tutorial, [])
+            mtw_list = test_suite.max_time_windows.get(tutorial, [])
+            for i, (case, reference_result) in enumerate(zip(
+                    test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
+                max_time = max_times[i] if i < len(max_times) else None
+                max_time_windows = mtw_list[i] if i < len(mtw_list) else None
                 systemtests_to_run.add(
-                    Systemtest(tutorial, build_args, case, reference_result))
+                    Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows))
 
     reference_result_per_tutorial = {}
     current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

tools/tests/systemtests.py

@@ -58,10 +58,14 @@ def main():
         for test_suite in test_suites_to_execute:
             tutorials = test_suite.cases_of_tutorial.keys()
             for tutorial in tutorials:
-                for case, reference_result in zip(
-                        test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]):
+                max_times = test_suite.max_times.get(tutorial, [])
+                mtw_list = test_suite.max_time_windows.get(tutorial, [])
+                for i, (case, reference_result) in enumerate(zip(
+                        test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
+                    max_time = max_times[i] if i < len(max_times) else None
+                    max_time_windows = mtw_list[i] if i < len(mtw_list) else None
                     systemtests_to_run.append(
-                        Systemtest(tutorial, build_args, case, reference_result))
+                        Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows))
 
     if not systemtests_to_run:
         raise RuntimeError("Did not find any Systemtests to execute.")

tools/tests/systemtests/Systemtest.py

@@ -134,6 +134,8 @@ class Systemtest:
     arguments: SystemtestArguments
     case_combination: CaseCombination
     reference_result: ReferenceResult
+    max_time: float | None = None
+    max_time_windows: int | None = None
     params_to_use: Dict[str, str] = field(init=False)
     env: Dict[str, str] = field(init=False)
 
@@ -513,11 +515,35 @@ def __write_logs(self, stdout_data: List[str], stderr_data: List[str]):
         with open(self.system_test_dir / "stderr.log", 'w') as stderr_file:
             stderr_file.write("\n".join(stderr_data))
 
+    def __apply_max_time_override(self):
+        """Overwrite <max-time> or <max-time-windows> value in precice-config.xml."""
+        if self.max_time is None and self.max_time_windows is None:
+            return
+        config_path = self.system_test_dir / "precice-config.xml"
+        text = config_path.read_text()
+        new_text = text
+        if self.max_time is not None:
+            pattern = r'(<max-time\s+value=")[^"]*(")'
+            new_text, count = re.subn(pattern, rf'\g<1>{self.max_time}\2', new_text)
+            if count == 0:
+                logging.warning(f"No <max-time> tag found in {config_path}")
+            else:
+                logging.info(f"Overwrote <max-time> to {self.max_time} in {config_path}")
+        if self.max_time_windows is not None:
+            pattern = r'(<max-time-windows\s+value=")[^"]*(")'
+            new_text, count = re.subn(pattern, rf'\g<1>{self.max_time_windows}\2', new_text)
+            if count == 0:
+                logging.warning(f"No <max-time-windows> tag found in {config_path}")
+            else:
+                logging.info(f"Overwrote <max-time-windows> to {self.max_time_windows} in {config_path}")
+        config_path.write_text(new_text)
+
     def __prepare_for_run(self, run_directory: Path):
         """
         Prepares the run_directory with folders and datastructures needed for every systemtest execution
         """
         self.__copy_tutorial_into_directory(run_directory)
+        self.__apply_max_time_override()
         self.__copy_tools(run_directory)
         self.__put_gitignore(run_directory)
         host_uid, host_gid = self.__get_uid_gid()

tools/tests/systemtests/TestSuite.py

@@ -10,6 +10,8 @@ class TestSuite:
     name: str
     cases_of_tutorial: Dict[Tutorial, List[CaseCombination]]
     reference_results: Dict[Tutorial, List[ReferenceResult]]
+    max_times: Dict[Tutorial, list] = field(default_factory=dict)
+    max_time_windows: Dict[Tutorial, list] = field(default_factory=dict)
 
     def __repr__(self) -> str:
         return_string = f"Test suite: {self.name} contains:"
@@ -48,6 +50,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
             for test_suite_name in test_suites_raw:
                 case_combinations_of_tutorial = {}
                 reference_results_of_tutorial = {}
+                max_times_of_tutorial = {}
+                max_time_windows_of_tutorial = {}
                 # iterate over tutorials:
                 for tutorial_case in test_suites_raw[test_suite_name]['tutorials']:
                     tutorial = parsed_tutorials.get_by_path(tutorial_case['path'])
@@ -57,6 +61,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                     if tutorial not in case_combinations_of_tutorial:
                         case_combinations_of_tutorial[tutorial] = []
                         reference_results_of_tutorial[tutorial] = []
+                        max_times_of_tutorial[tutorial] = []
+                        max_time_windows_of_tutorial[tutorial] = []
 
                     all_case_combinations = tutorial.case_combinations
                     case_combination_requested = CaseCombination.from_string_list(
@@ -65,12 +71,20 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                         case_combinations_of_tutorial[tutorial].append(case_combination_requested)
                         reference_results_of_tutorial[tutorial].append(ReferenceResult(
                             tutorial_case['reference_result'], case_combination_requested))
+                        max_time_raw = tutorial_case.get('max_time', None)
+                        if max_time_raw is not None and (not isinstance(max_time_raw, (int, float)) or max_time_raw <= 0):
+                            raise ValueError(f"max_time must be a positive number, got {max_time_raw!r}")
+                        max_times_of_tutorial[tutorial].append(max_time_raw)
+                        mtw_raw = tutorial_case.get('max_time_windows', None)
+                        if mtw_raw is not None and (not isinstance(mtw_raw, int) or mtw_raw <= 0):
+                            raise ValueError(f"max_time_windows must be a positive integer, got {mtw_raw!r}")
+                        max_time_windows_of_tutorial[tutorial].append(mtw_raw)
                     else:
                         raise Exception(
                             f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}")
 
                 testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial,
-                                            reference_results_of_tutorial))
+                                            reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial))
 
         return cls(testsuites)