ft-orchestration jq command use no intermediate file

Author: pascal260303

tools/ft-orchestration/src/collector/jq.py

@@ -69,18 +69,12 @@ def __init__(self, executor: Executor, file: str):
 
         self._executor = executor
         self._file = stdout.strip()
-        tmp_file = path.join(self._rsync.get_data_directory(), "flows.json")
         self._cmd_json = f"ipfixcol2 -c {Path(self._conf_dir, self.CONFIG_FILE)}"
         fields = ", ".join(
             [f'.\\"{field}\\"' for field in CSV_HEADER_TO_ANALYZER_HEADER.keys()]
         )
-        header = ",".join(
-            [f'"{field}"' for field in CSV_HEADER_TO_ANALYZER_HEADER.keys()]
-        )
         self._cmd_csv = f"""set -e
-ipfixcol2 -c {Path(self._conf_dir, self.CONFIG_FILE)} > {tmp_file}
-echo {header}
-cat {tmp_file} | jq -r "
+ipfixcol2 -c {Path(self._conf_dir, self.CONFIG_FILE)} | jq -r "
   .[\\"iana:sourceIPAddress\\"] = (.\\"iana:sourceIPv4Address\\" // .\\"iana:sourceIPv6Address\\") |
   .[\\"iana:destinationIPAddress\\"] = (.\\"iana:destinationIPv4Address\\" // .\\"iana:destinationIPv6Address\\") |
   del(
@@ -89,9 +83,8 @@ def __init__(self, executor: Executor, file: str):
     .\\"iana:destinationIPv4Address\\",
     .\\"iana:destinationIPv6Address\\"
   ) |
-  [{fields}] | @csv
-"
-rm {tmp_file}"""
+  [{fields}] | @csv"
+"""
         """Reads fds file and output as json with ipfixcol2, then converts json with `jq` to csv\\
         In the csv output the columns `iana:sourceIPv4Address` and `iana:sourceIPv6Address` are merged to `iana:sourceIPAddress`\\
         The same is done for `iana:destinationIPAddress`
@@ -286,15 +279,17 @@ def save_csv(self, csv_file: str):
         rsync = Rsync(self._executor)
         filename = path.basename(csv_file)
         tmp_file = path.join(rsync.get_data_directory(), filename)
+        header = ",".join(
+            [f'"{field}"' for field in CSV_HEADER_TO_ANALYZER_HEADER.keys()]
+        )
 
         logging.getLogger().info(
             "Preparing CSV output by calling ipfixcol2 + jq command..."
         )
         start = time.time()
+        Tool(f"echo '{header}' > {tmp_file}", executor=self._executor).run()
         # write csv
-        stdout, stderr = Tool(
-            f"({self._cmd_csv}) >> {tmp_file}", executor=self._executor
-        ).run()
+        Tool(f"({self._cmd_csv}) >> {tmp_file}", executor=self._executor).run()
         end = time.time()
         logging.getLogger().info("CSV output saved in %.2f seconds.", (end - start))