ft-orchestration save logs on tests with errors

Author: pascal260303

tools/ft-orchestration/src/collector/ipfixcol2.py

@@ -9,6 +9,7 @@
 """
 
 import logging
+import os
 import shutil
 import tempfile
 import xml.etree.ElementTree as ET
@@ -301,13 +302,18 @@ def download_logs(self, directory: str):
         directory : str
             Path to a local directory where logs should be stored.
         """
+        if self._executor.is_running():
+            self._executor.terminate()
 
         for log_file in self._prepare_logs():
             try:
                 self._rsync.pull_path(log_file, directory)
             except RsyncException as err:
                 logging.getLogger().warning("%s", err)
-        shutil.move(self._log_file, directory)
+        if not os.path.exists(
+            os.path.join(directory, os.path.basename(self._log_file))
+        ):
+            shutil.move(self._log_file, directory)
 
     def cleanup(self):
         """Delete working directory."""

tools/ft-orchestration/src/probe/pidstat.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import shutil
 import time
 from src.probe.interface import HostStats
 from lbr_testsuite.executable import (
@@ -98,6 +99,10 @@ def stop(self):
             pass
 
     def get_csv(self, output_dir) -> os.PathLike:
+        if self.local_file:
+            if os.path.dirname(self.local_file) == output_dir:
+                return self.local_file
+            return shutil.copy(self.local_file, output_dir)
         try:
             self.local_file = self._rsync.pull_path(self._outfile, output_dir)
             return self.local_file

tools/ft-orchestration/tests/simulation/test_simulation_general.py

@@ -345,11 +345,14 @@ def finalizer_download_logs():
         generator_config=generator_conf,
     )
 
-    # method stats blocks until traffic is sent
-    stats = generator_instance.stats()
-
-    probe_instance.stop()
-    collector_instance.stop()
+    try:
+        # method stats blocks until traffic is sent
+        stats = generator_instance.stats()
+        collector_instance.stop()
+        probe_instance.stop()
+    except Exception as e:
+        finalizer_download_logs()
+        raise e
 
     flows_file = os.path.join(tmp_dir, "flows.csv")
 

tools/ft-orchestration/tests/simulation/test_simulation_overload.py

@@ -293,11 +293,14 @@ def finalizer_download_logs():
         generator_config=generator_conf,
     )
 
-    # method stats blocks until traffic is sent
-    stats = generator_instance.stats()
-
-    probe_instance.stop()
-    collector_instance.stop()
+    try:
+        # method stats blocks until traffic is sent
+        stats = generator_instance.stats()
+        probe_instance.stop()
+        collector_instance.stop()
+    except Exception as e:
+        finalizer_download_logs()
+        raise e
 
     flows_file = os.path.join(tmp_dir, "flows.csv")
 

tools/ft-orchestration/tests/simulation/test_simulation_threshold.py

@@ -241,12 +241,15 @@ def run_single_test(loops: int, speed: MbpsSpeed) -> tuple[bool, StatisticalRepo
             generator_config=generator_conf,
         )
 
-        # method stats blocks until traffic is sent
-        stats = generator_instance.stats()
-        logging.getLogger().info("stats: %s", stats)
-
-        probe_instance.stop()
-        collector_instance.stop()
+        try:
+            # method stats blocks until traffic is sent
+            stats = generator_instance.stats()
+            logging.getLogger().info("stats: %s", stats)
+            probe_instance.stop()
+            collector_instance.stop()
+        except Exception as e:
+            finalizer_download_logs()
+            raise e
 
         # get flows.csv and replicated reference file in parallel
         with EXECUTOR_CLS() as parallel_executor: