Fix tests to work with out of source builds. (#1926)

* Update Packet++ tests to work with out of source builds.

* Updated Pcap++ to work with out of source builds.

* Lint

* Fix unix run_tests.

* args fixes.

* fix typo

* updated build dir name.

* changed build dir to a flag.

* Fix path errors by resolving to absolute path before sending to subprocess.

* Lint

* Update github actions to utilize working_directory.

* Fix windivert to copy to binary directory.

* Refactory run_tests_windows to mimic run_tests.

* Remove duplicate find_interface call.

* Add build dir to windows ci runs.

* Fix missed Bin.

* Fix f-string ambiguity.

* Make path absolute before passing to subprocess.

* Fix f-string.

* Fix cmd argument,

* Fix run command.

* Update mingw run_tests args.

* Fix windivert args.

* Try converting path to string before passing to subprocess.run()?

* Changed subprocess.run to use non-shell environment and utilize integrated success check.

* Simplify run_tests_windows.

* Add overrides for executable locations to allow easier local testing.

* Update Windows coverage results location as they are now generated in the build directory.

* Remove shell flag from subprocess call.

* Add __future__ annotations to enable lazy annotation evaluation.

* Lint

* Add logging info to help debugging.

* Add .exe to windows exe paths.

* Fix typo.

* Add overrides for Visual Studio as its is a multi-config generator.

* Fixup coverage report location.

* Fix run.

* Revert shell.

* Fixup Popen arguments.

* Pass shell=True.

* Fix command line option generation for include tests and skip tests.

* Lint

* Fix command.

* Lint.

* Update run_tests_windows from base.

* Add factory for tests lists. Fix duplicate TCP command.

* Update run commands.

* Remove hard coded path.

* Lint

* Update new CI job.

* Set logging to always log info.

Author: Dimi1010

.github/workflows/build_and_test.yml

@@ -131,7 +131,7 @@ jobs:
       - name: Test PcapPlusPlus
         run: |
           . .venv/bin/activate
-          python3 ci/run_tests/run_tests.py --interface eth0 ${{ matrix.additional-test-flags }}
+          python3 ci/run_tests/run_tests.py --interface eth0 ${{ matrix.additional-test-flags }} --build-dir "$BUILD_DIR"
 
       - name: Test Examples
         run: |
@@ -216,7 +216,7 @@ jobs:
       - name: Test PcapPlusPlus
         run: |
           . .venv/bin/activate
-          python3 ci/run_tests/run_tests.py --interface eth0 --test-suites "packet"
+          python3 ci/run_tests/run_tests.py --interface eth0 --test-suites "packet" --build-dir "$BUILD_DIR"
 
       - name: Check installation
         run: |
@@ -286,7 +286,7 @@ jobs:
       - name: Test PcapPlusPlus
         run: |
           . .venv/bin/activate
-          python3 ci/run_tests/run_tests.py --interface eth0
+          python3 ci/run_tests/run_tests.py --interface eth0 --build-dir "$BUILD_DIR"
 
       - name: Check installation
         run: |
@@ -347,9 +347,8 @@ jobs:
 
       - name: Test Packet++
         if: env.avx512 == 'true'
-        run: |
-          cd Tests/Packet++Test
-          Bin/Packet++Test
+        run: ./Packet++Test
+        working-directory: ${{ github.workspace }}/${{ env.BUILD_DIR }}/Tests/Packet++Test
 
       - name: Tests skipped (no AVX-512 hardware support)
         if: env.avx512 == 'false'
@@ -403,14 +402,12 @@ jobs:
         run: cmake --build "$BUILD_DIR" -j
 
       - name: Test Packet++
-        run: |
-          cd Tests/Packet++Test
-          Bin/Packet++Test
+        run: ./Packet++Test
+        working-directory: ${{ github.workspace }}/${{ env.BUILD_DIR }}/Tests/Packet++Test
 
       - name: Test Pcap++
-        run: |
-          cd Tests/Pcap++Test
-          Bin/Pcap++Test -n
+        run: ./Pcap++Test -n
+        working-directory: ${{ github.workspace }}/${{ env.BUILD_DIR }}/Tests/Pcap++Test
 
       - name: Check installation
         run: |
@@ -505,7 +502,7 @@ jobs:
         if: ${{ matrix.host-arch == matrix.arch }}
         run: |
           . .venv/bin/activate
-          python ci/run_tests/run_tests.py --interface en0
+          python ci/run_tests/run_tests.py --interface en0 --build-dir "$BUILD_DIR"
 
       - name: Test Examples
         if: ${{ matrix.host-arch == matrix.arch }}
@@ -645,7 +642,7 @@ jobs:
       - name: Test PcapPlusPlus
         run: |
           python -m pip install -r ci\run_tests\requirements.txt
-          python ci\run_tests\run_tests_windows.py
+          python ci\run_tests\run_tests_windows.py --build-dir $env:BUILD_DIR
 
       - name: Test Examples
         run: |
@@ -730,6 +727,8 @@ jobs:
           echo "ZSTD_HOME_PARAM=-z C:\zstd" >> $env:GITHUB_ENV
         if: matrix.use_zstd == true
 
+        # The visual studio generator is a multi-config generator. This means that the executable will be in a configuration subfolder (e.g., Debug or Release),
+        # relative to the target folder. Example: $(BuildDir)/Test/Packet++Test/Debug/Packet++Test.exe
       - name: Configure PcapPlusPlus
         run: cmake -A ${{ matrix.arch }} -G "${{ matrix.platform }}" -DPCAP_ROOT=${{ env.PCAP_SDK_DIR }} -S . -B "$env:BUILD_DIR"
 
@@ -742,7 +741,7 @@ jobs:
       - name: Test PcapPlusPlus
         run: |
           python -m pip install -r ci\run_tests\requirements.txt
-          python ci\run_tests\run_tests_windows.py --coverage
+          python ci\run_tests\run_tests_windows.py --coverage --build-dir $env:BUILD_DIR --packet-test-exe "Tests/Packet++Test/Debug/Packet++Test.exe" --pcap-test-exe "Tests/Pcap++Test/Debug/Pcap++Test.exe"
 
       - name: Test Examples
         run: |
@@ -754,7 +753,7 @@ jobs:
       - name: Upload Coverage Results
         uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
         with:
-          files: ./Tests/Pcap++Test/Pcap++Coverage.xml,./Tests/Packet++Test/Packet++Coverage.xml
+          files: ./Dist/Tests/Pcap++Test/Debug/Pcap++Coverage.xml,./Dist/Tests/Packet++Test/Debug/Packet++Coverage.xml
           flags: ${{ matrix.os }},unittest,${{ matrix.pcap_lib }}
           fail_ci_if_error: false
           verbose: true
@@ -781,6 +780,8 @@ jobs:
         shell: pwsh
         run: .\ci\install_windivert.ps1 -Target "$env:WINDIVERT_DIR"
 
+        # The visual studio generator is a multi-config generator. This means that the executable will be in a configuration subfolder (e.g., Debug or Release),
+        # relative to the target folder. Example: $(BuildDir)/Test/Packet++Test/Debug/Packet++Test.exe
       - name: Configure PcapPlusPlus (WinDivert)
         run: cmake -A x64 -G "Visual Studio 17 2022" -DPCAP_ROOT=${{ env.PCAP_SDK_DIR }} -DPCAPPP_USE_WINDIVERT=ON -DWinDivert_ROOT=${{ env.WINDIVERT_DIR }} -S . -B "$env:BUILD_DIR"
 
@@ -790,14 +791,17 @@ jobs:
       - name: Install tcpreplay
         run: ci\install_tcpreplay.bat
 
-      - name: Test PcapPlusPlus
+      - name: Copy WinDivert files to test folder
         shell: pwsh
         run: |
-          Copy-Item "${{ env.WINDIVERT_DIR }}\x64\WinDivert.dll" "Tests\Pcap++Test\Bin\WinDivert.dll" -Force
-          Copy-Item "${{ env.WINDIVERT_DIR }}\x64\WinDivert64.sys" "Tests\Pcap++Test\Bin\WinDivert64.sys" -Force
+          Copy-Item "${{ env.WINDIVERT_DIR }}\x64\WinDivert.dll" "$env:BUILD_DIR\Tests\Pcap++Test\Debug\WinDivert.dll" -Force
+          Copy-Item "${{ env.WINDIVERT_DIR }}\x64\WinDivert64.sys" "$env:BUILD_DIR\Tests\Pcap++Test\Debug\WinDivert64.sys" -Force
 
+      - name: Test PcapPlusPlus
+        shell: pwsh
+        run: |
           python -m pip install -r ci\run_tests\requirements.txt
-          python ci\run_tests\run_tests_windows.py --include-tests windivert
+          python ci\run_tests\run_tests_windows.py --include-tests windivert --build-dir "$env:BUILD_DIR" --packet-test-exe "Tests/Packet++Test/Debug/Packet++Test.exe" --pcap-test-exe "Tests/Pcap++Test/Debug/Pcap++Test.exe"
 
   freebsd:
     runs-on: ubuntu-latest
@@ -831,7 +835,7 @@ jobs:
             echo "Testing PcapPlusPlus"
             python -m ensurepip
             python -m pip install -r ci/run_tests/requirements.txt
-            python ci/run_tests/run_tests.py --interface "$interface_name"
+            python ci/run_tests/run_tests.py --interface "$interface_name" --build-dir Dist
 
             echo "Testing PcapPlusPlus examples"
             cd Tests/ExamplesTest
@@ -964,7 +968,7 @@ jobs:
       - name: Test PcapPlusPlus
         run: |
           . .venv/bin/activate
-          python ci/run_tests/run_tests.py --interface eth0 --use-sudo --pcap-test-args="-t xdp"
+          python ci/run_tests/run_tests.py --interface eth0 --use-sudo --pcap-test-args="-t xdp" --build-dir "$BUILD_DIR"
 
       - name: Create Cobertura Report
         run: |

Tests/Packet++Test/CMakeLists.txt

@@ -63,12 +63,31 @@ if(WIN32)
   target_compile_definitions(Packet++Test PUBLIC WIN32_LEAN_AND_MEAN)
 endif()
 
+target_include_directories(Packet++Test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
+
 if(MSVC)
   # This executable requires getopt.h not available on VStudio
   target_link_libraries(Packet++Test PRIVATE Getopt-for-Visual-Studio)
 endif()
 
-set_property(TARGET Packet++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
-set_property(TARGET Packet++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
-set_property(TARGET Packet++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
-add_test(NAME Packet++Test COMMAND $<TARGET_FILE:Packet++Test> WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/)
+add_custom_command(
+  TARGET Packet++Test
+  POST_BUILD
+  COMMAND
+    ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/PacketExamples
+    $<TARGET_FILE_DIR:Packet++Test>/PacketExamples
+)
+
+add_custom_command(
+  TARGET Packet++Test
+  POST_BUILD
+  COMMAND
+    ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/../../3rdParty/OUIDataset/PCPP_OUIDataset.json"
+    "$<TARGET_FILE_DIR:Packet++Test>/PCPP_OUIDataset.json"
+)
+
+# set_property(TARGET Packet++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
+# set_property(TARGET Packet++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
+# set_property(TARGET Packet++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
+# add_test(NAME Packet++Test COMMAND $<TARGET_FILE:Packet++Test> WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/)
+add_test(NAME Packet++Test COMMAND $<TARGET_FILE:Packet++Test> WORKING_DIRECTORY $<TARGET_FILE_DIR:Packet++Test>)

Tests/Packet++Test/Tests/EthAndArpTests.cpp

@@ -15,8 +15,7 @@ using pcpp_tests::utils::createPacketFromHexResource;
 PTF_TEST_CASE(OUILookup)
 {
 	pcpp::OUILookup lookupEngineJson;
-	PTF_ASSERT_GREATER_THAN(lookupEngineJson.initOUIDatabaseFromJson("../../3rdParty/OUIDataset/PCPP_OUIDataset.json"),
-	                        0);
+	PTF_ASSERT_GREATER_THAN(lookupEngineJson.initOUIDatabaseFromJson("PCPP_OUIDataset.json"), 0);
 
 	PTF_ASSERT_EQUAL(lookupEngineJson.getVendorName("aa:aa:aa:aa:aa:aa"), "Unknown");
 	// CIDR 36

Tests/Packet++Test/main.cpp

@@ -3,8 +3,8 @@
 #include "PcppTestFrameworkRun.h"
 #include "TestDefinition.h"
 #include "Logger.h"
-#include "../../Tests/Packet++Test/Utils/TestUtils.h"
 #include "Resources.h"
+#include "Utils/TestUtils.h"
 
 static struct option PacketTestOptions[] = {
 	{ "include-tags",        required_argument, nullptr, 't' },

Tests/Pcap++Test/CMakeLists.txt

@@ -27,8 +27,16 @@ if(MSVC)
   target_link_libraries(Pcap++Test PRIVATE Getopt-for-Visual-Studio)
 endif()
 
-set_property(TARGET Pcap++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
-set_property(TARGET Pcap++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
-set_property(TARGET Pcap++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
+add_custom_command(
+  TARGET Pcap++Test
+  POST_BUILD
+  COMMAND
+    ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}/PcapExamples"
+    "$<TARGET_FILE_DIR:Pcap++Test>/PcapExamples"
+)
 
-add_test(NAME Pcap++Test COMMAND $<TARGET_FILE:Pcap++Test> -n WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/)
+# set_property(TARGET Pcap++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
+# set_property(TARGET Pcap++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
+# set_property(TARGET Pcap++Test PROPERTY RUNTIME_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_SOURCE_DIR}/Bin")
+# add_test(NAME Pcap++Test COMMAND $<TARGET_FILE:Pcap++Test> -n WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/)
+add_test(NAME Pcap++Test COMMAND $<TARGET_FILE:Pcap++Test> -n WORKING_DIRECTORY $<TARGET_FILE_DIR:Pcap++Test>)

ci/run_tests/run_tests.py

@@ -3,6 +3,8 @@
 import os
 import subprocess
 import argparse
+from pathlib import Path
+from dataclasses import dataclass
 from contextlib import contextmanager
 from scapy.all import get_if_addr
 
@@ -21,30 +23,44 @@ def tcp_replay_worker(interface: str, tcpreplay_dir: str):
         tcpreplay_proc.kill()
 
 
-def run_packet_tests(args: list[str], use_sudo: bool):
-    cmd_line = ["sudo"] if use_sudo else []
-    cmd_line += [os.path.join("Bin", "Packet++Test"), *args]
+@dataclass
+class Runner:
+    build_dir: Path
+    use_sudo: bool = False
+    packet_test_path = Path("Tests", "Packet++Test", "Packet++Test")
+    pcap_test_path = Path("Tests", "Pcap++Test", "Pcap++Test")
 
-    completed_process = subprocess.run(cmd_line, cwd="Tests/Packet++Test")
+    def run_packet_tests(self, args: list[str]):
+        exe_path = self.build_dir / self.packet_test_path
+        work_dir = exe_path.parent
 
-    if completed_process.returncode != 0:
-        raise RuntimeError(f"Error while executing Packet++ tests: {completed_process}")
+        cmd_line = ["sudo"] if self.use_sudo else []
+        cmd_line += [str(exe_path.absolute()), *args]
 
+        completed_process = subprocess.run(cmd_line, cwd=str(work_dir))
 
-def run_pcap_tests(interface: str, tcpreplay_dir: str, args: list[str], use_sudo: bool):
-    ip_address = get_if_addr(interface)
-    print(f"IP address is: {ip_address}")
-
-    with tcp_replay_worker(interface, tcpreplay_dir):
-        cmd_line = ["sudo"] if use_sudo else []
-        cmd_line += [os.path.join("Bin", "Pcap++Test"), "-i", ip_address, *args]
-
-        completed_process = subprocess.run(cmd_line, cwd="Tests/Pcap++Test")
         if completed_process.returncode != 0:
             raise RuntimeError(
-                f"Error while executing Pcap++ tests: {completed_process}"
+                f"Error while executing Packet++ tests: {completed_process}"
             )
 
+    def run_pcap_tests(self, interface: str, tcpreplay_dir: str, args: list[str]):
+        ip_address = get_if_addr(interface)
+        print(f"IP address is: {ip_address}")
+
+        exe_path = self.build_dir / self.pcap_test_path
+        work_dir = exe_path.parent
+
+        cmd_line = ["sudo"] if self.use_sudo else []
+        cmd_line += [str(exe_path.absolute()), "-i", ip_address, *args]
+
+        with tcp_replay_worker(interface, tcpreplay_dir):
+            completed_process = subprocess.run(cmd_line, cwd=str(work_dir))
+            if completed_process.returncode != 0:
+                raise RuntimeError(
+                    f"Error while executing Pcap++ tests: {completed_process}"
+                )
+
 
 def main():
     parser = argparse.ArgumentParser()
@@ -78,17 +94,19 @@ def main():
         default=None,
         help="tcpreplay directory",
     )
+    parser.add_argument(
+        "--build-dir", type=str, default=os.getcwd(), help="path to the build directory"
+    )
     args = parser.parse_args()
 
+    runner = Runner(build_dir=Path(args.build_dir), use_sudo=args.use_sudo)
+
     if "packet" in args.test_suites:
-        run_packet_tests(args.packet_test_args.split(), args.use_sudo)
+        runner.run_packet_tests(args.packet_test_args.split())
 
     if "pcap" in args.test_suites:
-        run_pcap_tests(
-            args.interface,
-            args.tcpreplay_dir,
-            args.pcap_test_args.split(),
-            args.use_sudo,
+        runner.run_pcap_tests(
+            args.interface, args.tcpreplay_dir, args.pcap_test_args.split()
         )