ft-analyzer optimize replication by avoiding deep copying and reading unnecessary columns

Author: pascal260303

run_tests_nohup.sh

@@ -12,4 +12,6 @@ echo "📋 Starting tests with nohup... output in ${LOG}"
 . .venv/bin/activate && pytest "$@" &
 wait %1 # wait for pytest to finish
 NEWEST_DIR=$(ls -d logs/[0-9]*/ | sort -r | head -n1)
-mv "logs/report.html" "${NEWEST_DIR}"
+if ! [ -e "${NEWEST_DIR}/report.html" ]; then
+  mv "logs/report.html" "${NEWEST_DIR}"
+fi

tools/ft-analyzer/ftanalyzer/replicator/flow_replicator.py

@@ -281,7 +281,11 @@ def replicate(
         try:
             for chunk in pd.read_csv(
                 input_file,
-                dtype=self.CSV_COLUMN_TYPES,
+                usecols=["START_TIME", "END_TIME"],
+                dtype={
+                    "START_TIME": np.uint64,
+                    "END_TIME": np.uint64,
+                },
                 chunksize=chunksize,
             ):
                 start_min = chunk["START_TIME"].min()
@@ -521,7 +525,7 @@ def _process_single_loop(
         if self._config.loop.dstip:
             dstip_offset += loop_n * self._config.loop.dstip.value
 
-        flows = self._flows.copy()
+        flows = self._flows.copy(deep=False)
         flows["START_TIME"] = time_offset + flows["_START_OFFSET"]
         flows["END_TIME"] = flows["START_TIME"] + flows["_FLOW_LEN"]