net: NAD live update VM connectivity tests (#4976)

##### What this PR does / why we need it:
Implements two NAD live-update tests for Linux bridge:
- `test_connectivity`: after iface-1 is moved from VLAN-A to VLAN-B,
verifies the under-test VM gains TCP connectivity on VLAN-B and loses it
on VLAN-A.
- `test_two_networks`: applies a single atomic patch swapping both
secondary networks simultaneously and verifies each interface reaches
its new VLAN.

Adds bind_dev support to TcpServer and VMTcpClient (--bind-dev /
SO_BINDTODEVICE) to force iperf3 traffic through a specific interface,
bypassing ECMP routing ambiguity when two secondary interfaces
  share the same subnet.

Adds poll_tcp_connectivity to tests/network/libs/connectivity.py: a
retrying TCP connectivity probe used by the baseline connectivity
fixture and the tests.


##### Which issue(s) this PR fixes: -

##### Special notes for reviewer: 
Depends on
#4962 -
only review the last 3 commits

##### jira-ticket: https://redhat.atlassian.net/browse/CNV-80573
<!-- full-ticket-url needs to be provided. This would add a link to the
pull request to the jira and close it when the pull request is merged
If the task is not tracked by a Jira ticket, just write "NONE".
-->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Enhanced traffic generator tools to support binding network traffic to
specific guest network interfaces.
* Added TCP connectivity polling capabilities for network testing
scenarios.

* **Tests**
* Refactored VLAN reference-change tests with improved fixture-driven
coverage.
* Introduced new test helpers and fixtures for validating
multi-interface network connectivity.

<!-- review_stack_entry_start -->

[![Review Change
Stack](https://storage.googleapis.com/coderabbit_public_assets/review-stack-in-coderabbit-ui.svg)](https://app.coderabbit.ai/change-stack/RedHatQE/openshift-virtualization-tests/pull/4976?utm_source=github_walkthrough&utm_medium=github&utm_campaign=change_stack)

<!-- review_stack_entry_end -->

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: rnetser

libs/net/traffic_generator.py

@@ -53,18 +53,22 @@ class TcpServer:
         vm (BaseVirtualMachine): The virtual machine where the server runs.
         port (int): The port on which the server listens for client connections.
         bind_ip (str): The IP address to bind the server to (optional).
+        bind_dev (str): Guest network device to bind the server socket to via SO_BINDTODEVICE
+            (e.g. "eth1"). Forces responses out this interface, bypassing ECMP routing.
     """
 
     def __init__(
         self,
         vm: BaseVirtualMachine,
         port: int,
         bind_ip: str | None = None,
+        bind_dev: str | None = None,
     ):
         self._vm = vm
         self._port = port
         self._cmd = f"{_IPERF_BIN} --server --port {self._port} --one-off"
         self._cmd += f" --bind {bind_ip}" if bind_ip else ""
+        self._cmd += f" --bind-dev {bind_dev}" if bind_dev else ""
 
     def __enter__(self) -> "TcpServer":
         self._vm.console(
@@ -100,6 +104,8 @@ class VMTcpClient(BaseTcpClient):
         server_port (int): The port on which the server listens for connections.
         maximum_segment_size (int): Define explicitly the TCP payload size (in bytes).
                                     Default value is 0 (do not change mss).
+        bind_dev (str): Guest network device to bind the client socket to via SO_BINDTODEVICE
+            (e.g. "eth1"). Forces traffic out this interface, bypassing ECMP routing.
     """
 
     def __init__(
@@ -108,9 +114,11 @@ def __init__(
         server_ip: str,
         server_port: int,
         maximum_segment_size: int = 0,
+        bind_dev: str | None = None,
     ):
         super().__init__(server_ip=server_ip, server_port=server_port)
         self._vm = vm
+        self._cmd += f" --bind-dev {bind_dev}" if bind_dev else ""
         self._cmd += f" --set-mss {maximum_segment_size}" if maximum_segment_size else ""
 
     def __enter__(self) -> "VMTcpClient":

tests/network/l2_bridge/nad_ref_change/conftest.py

@@ -4,17 +4,19 @@
 from kubernetes.dynamic import DynamicClient
 from ocp_resources.namespace import Namespace
 
-from libs.net.ip import random_cidr_addresses_by_family
+from libs.net.ip import filter_link_local_addresses, random_cidr_addresses_by_family
 from libs.net.netattachdef import CNIPluginBridgeConfig, NetConfig, NetworkAttachmentDefinition
-from libs.net.vmspec import wait_for_ifaces_status
+from libs.net.vmspec import lookup_iface_status, wait_for_ifaces_status
 from libs.vm.vm import BaseVirtualMachine
 from tests.network.l2_bridge.libl2bridge import LINUX_BRIDGE_IFACE_NAME_1, LINUX_BRIDGE_IFACE_NAME_2
 from tests.network.l2_bridge.nad_ref_change.lib_helpers import (
+    GUEST_IFACE_1,
+    GUEST_IFACE_2,
     NET_SEED,
     two_secondary_bridge_vm,
 )
 from tests.network.libs import nodenetworkconfigurationpolicy as libnncp
-from tests.network.libs.connectivity import ARP_ISOLATION_SYSCTL_CMD
+from tests.network.libs.connectivity import ARP_ISOLATION_SYSCTL_CMD, poll_tcp_connectivity
 
 
 @pytest.fixture(scope="module")
@@ -55,6 +57,36 @@ def bridge_nad_b(
         yield nad
 
 
+@pytest.fixture(scope="module")
+def ref_vm(
+    namespace: Namespace,
+    unprivileged_client: DynamicClient,
+    bridge_nad_a: NetworkAttachmentDefinition,
+    bridge_nad_b: NetworkAttachmentDefinition,
+) -> Generator[BaseVirtualMachine]:
+    iface_a_ips = random_cidr_addresses_by_family(net_seed=NET_SEED, host_address=1)
+    iface_b_ips = random_cidr_addresses_by_family(net_seed=NET_SEED, host_address=2)
+    with two_secondary_bridge_vm(
+        namespace=namespace.name,
+        name="ref-vm",
+        client=unprivileged_client,
+        nad_names=[bridge_nad_a.name, bridge_nad_b.name],
+        ip_addresses=[iface_a_ips, iface_b_ips],
+        iface_names=[LINUX_BRIDGE_IFACE_NAME_1, LINUX_BRIDGE_IFACE_NAME_2],
+        runcmd=ARP_ISOLATION_SYSCTL_CMD,
+    ) as vm:
+        vm.start(wait=True)
+        vm.wait_for_agent_connected()
+        wait_for_ifaces_status(
+            vm=vm,
+            ip_addresses_by_spec_net_name={
+                LINUX_BRIDGE_IFACE_NAME_1: [addr.split("/")[0] for addr in iface_a_ips],
+                LINUX_BRIDGE_IFACE_NAME_2: [addr.split("/")[0] for addr in iface_b_ips],
+            },
+        )
+        yield vm
+
+
 @pytest.fixture(scope="class")
 def under_test_vm_two_ifaces(
     namespace: Namespace,
@@ -82,3 +114,34 @@ def under_test_vm_two_ifaces(
             },
         )
         yield vm
+
+
+@pytest.fixture(scope="class")
+def baseline_connectivity(
+    under_test_vm_two_ifaces: BaseVirtualMachine,
+    ref_vm: BaseVirtualMachine,
+) -> None:
+    """Verify baseline connectivity before the NAD reference change.
+
+    Asserts that the under-test VM can reach the reference VM on VLAN-A (iface-1)
+    and cannot reach it on VLAN-B (iface-2) before any NAD update is applied.
+    """
+    for server_ip in filter_link_local_addresses(
+        ip_addresses=lookup_iface_status(vm=ref_vm, iface_name=LINUX_BRIDGE_IFACE_NAME_1).ipAddresses
+    ):
+        poll_tcp_connectivity(
+            client_vm=under_test_vm_two_ifaces,
+            server_vm=ref_vm,
+            server_ip=str(server_ip),
+            server_bind_dev=GUEST_IFACE_1,
+        )
+    for server_ip in filter_link_local_addresses(
+        ip_addresses=lookup_iface_status(vm=ref_vm, iface_name=LINUX_BRIDGE_IFACE_NAME_2).ipAddresses
+    ):
+        poll_tcp_connectivity(
+            client_vm=under_test_vm_two_ifaces,
+            server_vm=ref_vm,
+            server_ip=str(server_ip),
+            server_bind_dev=GUEST_IFACE_2,
+            expect_connectivity=False,
+        )

tests/network/l2_bridge/nad_ref_change/lib_helpers.py

@@ -15,6 +15,7 @@
 from libs.vm.vm import BaseVirtualMachine, add_volume_disk, cloudinitdisk_storage
 from tests.network.libs import cloudinit
 from tests.network.libs.cloudinit import primary_iface_cloud_init
+from tests.network.libs.connectivity import poll_tcp_connectivity
 
 NET_SEED: Final[int] = 0
 
@@ -23,6 +24,57 @@
 GUEST_IFACE_2: Final[str] = "eth2"
 
 
+def assert_connectivity(
+    client_vm: BaseVirtualMachine,
+    server_vm: BaseVirtualMachine,
+    server_ip: str,
+    server_bind_dev: str,
+    client_bind_dev: str,
+) -> None:
+    """Assert TCP connectivity from client to server for a single IP address.
+
+    Args:
+        client_vm: VM initiating the connection.
+        server_vm: VM accepting the connection.
+        server_ip: IP address to connect to.
+        server_bind_dev: Guest device to bind the iperf3 server to (bypasses ECMP).
+        client_bind_dev: Guest device to bind the iperf3 client to (bypasses ECMP).
+    """
+    poll_tcp_connectivity(
+        client_vm=client_vm,
+        server_vm=server_vm,
+        server_ip=server_ip,
+        client_bind_dev=client_bind_dev,
+        server_bind_dev=server_bind_dev,
+    )
+
+
+def assert_no_connectivity(
+    client_vm: BaseVirtualMachine,
+    server_vm: BaseVirtualMachine,
+    server_ip: str,
+    server_bind_dev: str,
+    client_bind_dev: str,
+) -> None:
+    """Assert no TCP connectivity from client to server for a single IP address.
+
+    Args:
+        client_vm: VM initiating the connection.
+        server_vm: VM accepting the connection.
+        server_ip: IP address to connect to.
+        server_bind_dev: Guest device to bind the iperf3 server to (bypasses ECMP).
+        client_bind_dev: Guest device to bind the iperf3 client to (bypasses ECMP).
+    """
+    poll_tcp_connectivity(
+        client_vm=client_vm,
+        server_vm=server_vm,
+        server_ip=server_ip,
+        client_bind_dev=client_bind_dev,
+        server_bind_dev=server_bind_dev,
+        expect_connectivity=False,
+    )
+
+
 def update_nad_references(vm: BaseVirtualMachine, nad_name_by_net: dict[str, str]) -> None:
     """Update secondary network NAD references and wait for the change to be fully applied.
 

tests/network/l2_bridge/nad_ref_change/test_nad_ref_change.py

@@ -13,13 +13,19 @@
 
 import pytest
 
+from libs.net.ip import filter_link_local_addresses
 from libs.net.vmspec import lookup_iface_status
-from tests.network.l2_bridge.libl2bridge import LINUX_BRIDGE_IFACE_NAME_1
+from tests.network.l2_bridge.libl2bridge import LINUX_BRIDGE_IFACE_NAME_1, LINUX_BRIDGE_IFACE_NAME_2
 from tests.network.l2_bridge.nad_ref_change.lib_helpers import (
+    GUEST_IFACE_1,
+    GUEST_IFACE_2,
+    assert_connectivity,
+    assert_no_connectivity,
     update_nad_references,
 )
 
 
+@pytest.mark.usefixtures("baseline_connectivity")
 @pytest.mark.incremental
 class TestRunningVMLinuxBridgeVlanChange:
     """
@@ -80,7 +86,7 @@ def test_vm_state_iface_info_preserved(
 
         Expected:
             - Under-test VM remains running after the NAD reference change
-            - Guest first secondary interface MAC address, name, and IP addresses are the same before and after the
+            - All guest first secondary interface status fields are identical before and after the
               NAD reference change
         """
         iface_before = lookup_iface_status(vm=under_test_vm_two_ifaces, iface_name=LINUX_BRIDGE_IFACE_NAME_1)
@@ -95,7 +101,14 @@ def test_vm_state_iface_info_preserved(
         )
 
     @pytest.mark.polarion("CNV-15972")
-    def test_connectivity(self):
+    def test_connectivity(
+        self,
+        subtests,
+        under_test_vm_two_ifaces,
+        ref_vm,
+        bridge_nad_a,
+        bridge_nad_b,
+    ):
         """
         Test that the under-test VM has TCP connectivity to the reference VM on the new NAD-VLAN-B and no TCP
         connectivity on the old NAD-VLAN-A after the NAD reference change.
@@ -112,11 +125,38 @@ def test_connectivity(self):
             - Under-test VM eventually has TCP connectivity to the reference VM on NAD-VLAN-B
             - Under-test VM has no TCP connectivity to the reference VM on NAD-VLAN-A
         """
-
-    test_connectivity.__test__ = False
+        for server_ip in filter_link_local_addresses(
+            ip_addresses=lookup_iface_status(vm=ref_vm, iface_name=LINUX_BRIDGE_IFACE_NAME_2).ipAddresses
+        ):
+            with subtests.test(msg=f"IPv{server_ip.version} on {bridge_nad_b.name}"):
+                assert_connectivity(
+                    client_vm=under_test_vm_two_ifaces,
+                    server_vm=ref_vm,
+                    server_ip=str(server_ip),
+                    server_bind_dev=GUEST_IFACE_2,
+                    client_bind_dev=GUEST_IFACE_1,
+                )
+        for server_ip in filter_link_local_addresses(
+            ip_addresses=lookup_iface_status(vm=ref_vm, iface_name=LINUX_BRIDGE_IFACE_NAME_1).ipAddresses
+        ):
+            with subtests.test(msg=f"IPv{server_ip.version} on {bridge_nad_a.name}"):
+                assert_no_connectivity(
+                    client_vm=under_test_vm_two_ifaces,
+                    server_vm=ref_vm,
+                    server_ip=str(server_ip),
+                    server_bind_dev=GUEST_IFACE_1,
+                    client_bind_dev=GUEST_IFACE_1,
+                )
 
     @pytest.mark.polarion("CNV-15946")
-    def test_two_networks(self):
+    def test_two_networks(
+        self,
+        subtests,
+        under_test_vm_two_ifaces,
+        ref_vm,
+        bridge_nad_a,
+        bridge_nad_b,
+    ):
         """
         Test that both secondary Linux bridge networks on a running VM can have their VLANs
         changed in a single patch, with each network switching to a different VLAN.
@@ -140,8 +180,36 @@ def test_two_networks(self):
             - Under-test VM first secondary network eventually has TCP connectivity to the reference VM on NAD-VLAN-A
             - Under-test VM second secondary network eventually has TCP connectivity to the reference VM on NAD-VLAN-B
         """
+        update_nad_references(
+            vm=under_test_vm_two_ifaces,
+            nad_name_by_net={
+                LINUX_BRIDGE_IFACE_NAME_1: bridge_nad_a.name,
+                LINUX_BRIDGE_IFACE_NAME_2: bridge_nad_b.name,
+            },
+        )
 
-    test_two_networks.__test__ = False
+        for server_ip in filter_link_local_addresses(
+            ip_addresses=lookup_iface_status(vm=ref_vm, iface_name=LINUX_BRIDGE_IFACE_NAME_1).ipAddresses
+        ):
+            with subtests.test(msg=f"IPv{server_ip.version} on {bridge_nad_a.name}"):
+                assert_connectivity(
+                    client_vm=under_test_vm_two_ifaces,
+                    server_vm=ref_vm,
+                    server_ip=str(server_ip),
+                    server_bind_dev=GUEST_IFACE_1,
+                    client_bind_dev=GUEST_IFACE_1,
+                )
+        for server_ip in filter_link_local_addresses(
+            ip_addresses=lookup_iface_status(vm=ref_vm, iface_name=LINUX_BRIDGE_IFACE_NAME_2).ipAddresses
+        ):
+            with subtests.test(msg=f"IPv{server_ip.version} on {bridge_nad_b.name}"):
+                assert_connectivity(
+                    client_vm=under_test_vm_two_ifaces,
+                    server_vm=ref_vm,
+                    server_ip=str(server_ip),
+                    server_bind_dev=GUEST_IFACE_2,
+                    client_bind_dev=GUEST_IFACE_2,
+                )
 
 
 @pytest.mark.polarion("CNV-15947")

tests/network/libs/connectivity.py

@@ -1,6 +1,11 @@
 import ipaddress
 from typing import Final
 
+from timeout_sampler import TimeoutExpiredError, retry
+
+from libs.net.traffic_generator import IPERF_SERVER_PORT, TcpServer, VMTcpClient
+from libs.vm.vm import BaseVirtualMachine
+
 ARP_ISOLATION_SYSCTL_CMD: Final[list[str]] = [
     # Only answer ARP for the IP assigned to the receiving interface —
     # prevents eth1 from responding to ARP for eth2's IP when queried from the same VLAN.
@@ -26,3 +31,38 @@ def build_ping_command(dst_ip: str, count: int, timeout: int) -> str:
     ip = ipaddress.ip_address(address=dst_ip)
     ping_ipv6_flag = " -6" if ip.version == 6 else ""
     return f"ping{ping_ipv6_flag} {dst_ip} -c {count} -w {timeout}"
+
+
+@retry(wait_timeout=60, sleep=5, exceptions_dict={})
+def poll_tcp_connectivity(
+    client_vm: BaseVirtualMachine,
+    server_vm: BaseVirtualMachine,
+    server_ip: str,
+    client_bind_dev: str | None = None,
+    server_bind_dev: str | None = None,
+    expect_connectivity: bool = True,
+) -> bool:
+    """Poll TCP connectivity (or its absence) between two VMs, retrying until the expected state is reached.
+
+    Args:
+        client_vm: VM initiating the TCP connection.
+        server_vm: VM running the iperf3 server.
+        server_ip: IP address the server binds to.
+        client_bind_dev: Guest network device name to force the client out (e.g. "eth1").
+            Bypasses ECMP routing when both secondary interfaces share the same subnet.
+        server_bind_dev: Guest network device name to force the server responses out (e.g. "eth1").
+            Bypasses ECMP routing on the server VM when it has multiple secondary interfaces.
+        expect_connectivity: When True polls until connectivity exists; when False polls until it does not.
+
+    Returns:
+        True when the observed reachability matches expect_connectivity.
+    """
+    try:
+        with TcpServer(vm=server_vm, port=IPERF_SERVER_PORT, bind_ip=server_ip, bind_dev=server_bind_dev):
+            with VMTcpClient(
+                vm=client_vm, server_ip=server_ip, server_port=IPERF_SERVER_PORT, bind_dev=client_bind_dev
+            ):
+                reachable = True
+    except TimeoutExpiredError:
+        reachable = False
+    return reachable if expect_connectivity else not reachable