ft-orchestration jq.py replace lbr_testsuite ssh execution with manual ssh command + redirect

Author: pascal260303

tools/ft-orchestration/src/collector/jq.py

@@ -5,6 +5,7 @@
 import time
 import xml.etree.ElementTree as ET
 
+import fabric
 import numpy as np
 from src.collector.interface import (
     CollectorOutputReaderException,
@@ -290,28 +291,29 @@ def save_csv(self, csv_file: str):
             Path to CSV file. Local file, CSV will be downloaded when collector running on remote.
         """
         header = CSV_HEADER_TO_ANALYZER_HEADER.values()
-
-        batch_size = 100_000
-        buffer = []
+        conn: fabric.Connection = self._executor._connection
+        user = conn.user
+        host = conn.host
+        kwargs = conn.connect_kwargs
+
+        if "key_filename" in kwargs:
+            key_filename = kwargs["key_filename"]
+            ssh_cmd = (
+                f"ssh -i {key_filename} {user}@{host} '{self._cmd_csv}' >> {csv_file}"
+            )
+        else:
+            password = kwargs["password"]
+            ssh_cmd = f"sshpass -p {password} ssh {user}@{host} '{self._cmd_csv}' >> {csv_file}"
 
         logging.getLogger().info(
             "Preparing CSV output by calling ipfixcol2 + jq command..."
         )
         start = time.time()
         # write csv
-        process = AsyncTool(self._cmd_csv, executor=self._executor)
-        process.run()
-
-        with open(csv_file, mode="w", buffering=1024**2) as f:
+        with open(csv_file, mode="w") as f:
             f.write(",".join(header) + "\n")
-            for line in process.stdout:
-                buffer.append(line + "\n")
-                if len(buffer) >= batch_size:
-                    f.writelines(buffer)
-                    buffer = []
-
-            if buffer:
-                f.writelines(buffer)
+
+        Tool(ssh_cmd).run()
 
         end = time.time()
         logging.getLogger().info("CSV output saved in %.2f seconds.", (end - start))