net, tests, stuntime: Wait for VMI affinity before migration

virt-controller and VMIM controller reconcile independently — if
migration is triggered before the VM controller propagates the template
affinity to the VMI, the target pod is created with stale scheduling
rules.

Wait for the VMI to reflect the updated affinity before migrating, and
assert post-migration node placement matches the expected affinity.

Assisted-by: Claude <noreply@anthropic.com>
Signed-off-by: Anat Wax <awax@redhat.com>

Author: Anatw

libs/vm/vm.py

@@ -10,6 +10,7 @@
 from ocp_resources.virtual_machine import VirtualMachine
 from ocp_resources.virtual_machine_instance import VirtualMachineInstance
 from pytest_testconfig import config as py_config
+from timeout_sampler import TimeoutSampler
 
 from libs.net.vmspec import VMInterfaceSpecNotFoundError
 from libs.vm.spec import (
@@ -143,6 +144,24 @@ def set_template_affinity(self, affinity: Affinity | None) -> None:
         patches = {self: {"spec": {"template": {"spec": {"affinity": template_affinity}}}}}
         ResourceEditor(patches=patches).update()
 
+    def wait_for_vmi_affinity(self, timeout: int = 10) -> None:
+        """Wait for the VMI to reflect the current template affinity.
+
+        Args:
+            timeout: Maximum seconds to wait for reconciliation.
+        """
+        template_affinity = self._spec.template.spec.affinity
+        expected_affinity = (
+            asdict(obj=template_affinity, dict_factory=self._filter_out_none_values) if template_affinity else None
+        )
+        for sample in TimeoutSampler(
+            wait_timeout=timeout,
+            sleep=1,
+            func=lambda: self.vmi.instance.to_dict()["spec"].get("affinity"),
+        ):
+            if sample == expected_affinity:
+                break
+
     @property
     def template_spec(self) -> VMISpec:
         return self._spec.template.spec

tests/network/l2_bridge/migration_stuntime/test_migration_stuntime.py

@@ -19,7 +19,16 @@
 import pytest
 
 from libs.vm.affinity import new_pod_affinity, new_pod_anti_affinity
-from tests.network.libs.stuntime import CLIENT_VM_LABEL, SERVER_VM_LABEL, STUNTIME_THRESHOLD_SECONDS, measure_stuntime
+from tests.network.libs.stuntime import (
+    CLIENT_VM_LABEL,
+    POD_AFFINITY_TYPE,
+    POD_ANTI_AFFINITY_TYPE,
+    SERVER_VM_LABEL,
+    STUNTIME_THRESHOLD_SECONDS,
+    assert_affinity_after_migration,
+    measure_stuntime,
+)
+from utilities.jira import is_jira_open
 from utilities.virt import migrate_vm_and_verify
 
 pytestmark = [pytest.mark.tier3]
@@ -71,7 +80,10 @@ def test_client_migrates_off_server_node(
             - Measured stuntime does not exceed the global threshold.
         """
         stuntime_client_vm.set_template_affinity(affinity=new_pod_anti_affinity(label=SERVER_VM_LABEL))
+        stuntime_client_vm.wait_for_vmi_affinity()
         migrate_vm_and_verify(vm=stuntime_client_vm, client=admin_client)
+        if is_jira_open(jira_id="CNV-90576"):
+            assert_affinity_after_migration(vm=stuntime_client_vm, expected_type=POD_ANTI_AFFINITY_TYPE)
         measured_stuntime = measure_stuntime(active_ping=l2_bridge_active_ping)
         assert measured_stuntime <= STUNTIME_THRESHOLD_SECONDS, (
             f"Stuntime {measured_stuntime}s exceeds threshold ({STUNTIME_THRESHOLD_SECONDS}s)"
@@ -130,7 +142,10 @@ def test_client_migrates_to_server_node(
             - Measured stuntime does not exceed the global threshold.
         """
         stuntime_client_vm.set_template_affinity(affinity=new_pod_affinity(label=SERVER_VM_LABEL))
+        stuntime_client_vm.wait_for_vmi_affinity()
         migrate_vm_and_verify(vm=stuntime_client_vm, client=admin_client)
+        if is_jira_open(jira_id="CNV-90576"):
+            assert_affinity_after_migration(vm=stuntime_client_vm, expected_type=POD_AFFINITY_TYPE)
         measured_stuntime = measure_stuntime(active_ping=l2_bridge_active_ping)
         assert measured_stuntime <= STUNTIME_THRESHOLD_SECONDS, (
             f"Stuntime {measured_stuntime}s exceeds threshold ({STUNTIME_THRESHOLD_SECONDS}s)"
@@ -160,7 +175,10 @@ def test_server_migrates_off_client_node(
             - Measured stuntime does not exceed the global threshold.
         """
         stuntime_server_vm.set_template_affinity(affinity=new_pod_anti_affinity(label=CLIENT_VM_LABEL))
+        stuntime_server_vm.wait_for_vmi_affinity()
         migrate_vm_and_verify(vm=stuntime_server_vm, client=admin_client)
+        if is_jira_open(jira_id="CNV-90576"):
+            assert_affinity_after_migration(vm=stuntime_server_vm, expected_type=POD_ANTI_AFFINITY_TYPE)
         measured_stuntime = measure_stuntime(active_ping=l2_bridge_active_ping)
         assert measured_stuntime <= STUNTIME_THRESHOLD_SECONDS, (
             f"Stuntime {measured_stuntime}s exceeds threshold ({STUNTIME_THRESHOLD_SECONDS}s)"
@@ -219,7 +237,10 @@ def test_server_migrates_to_client_node(
             - Measured stuntime does not exceed the global threshold.
         """
         stuntime_server_vm.set_template_affinity(affinity=new_pod_affinity(label=CLIENT_VM_LABEL))
+        stuntime_server_vm.wait_for_vmi_affinity()
         migrate_vm_and_verify(vm=stuntime_server_vm, client=admin_client)
+        if is_jira_open(jira_id="CNV-90576"):
+            assert_affinity_after_migration(vm=stuntime_server_vm, expected_type=POD_AFFINITY_TYPE)
         measured_stuntime = measure_stuntime(active_ping=l2_bridge_active_ping)
         assert measured_stuntime <= STUNTIME_THRESHOLD_SECONDS, (
             f"Stuntime {measured_stuntime}s exceeds threshold ({STUNTIME_THRESHOLD_SECONDS}s)"

tests/network/libs/stuntime.py

@@ -14,6 +14,8 @@
 SERVER_VM_LABEL: Final[tuple[str, str]] = (STUNTIME_LABEL_KEY, "server")
 CLIENT_VM_LABEL: Final[tuple[str, str]] = (STUNTIME_LABEL_KEY, "client")
 STUNTIME_THRESHOLD_SECONDS: Final[float] = 5.0
+POD_AFFINITY_TYPE: Final[str] = "podAffinity"
+POD_ANTI_AFFINITY_TYPE: Final[str] = "podAntiAffinity"
 STUNTIME_PING_LOG_PATH: Final[str] = "/tmp/stuntime-ping.log"
 PING_INTERVAL_SECONDS: Final[float] = 0.01
 DEFAULT_COMMAND_TIMEOUT_SECONDS: Final[int] = 10
@@ -141,3 +143,29 @@ def _compute_stuntime(lost_packets: int) -> float:
     stuntime = 0.0 if lost_packets == 0 else (lost_packets + 1) * PING_INTERVAL_SECONDS
     LOGGER.info(f"Stuntime: {stuntime:.2f}s (from {lost_packets} lost packets)")
     return stuntime
+
+
+def assert_affinity_after_migration(vm: BaseVirtualMachine, expected_type: str) -> None:
+    """Verify the migration target pod has the correct affinity after migration.
+
+    Detects the virt-controller race (CNV-90576) where the migration target pod is
+    created from a stale VMI snapshot, resulting in incorrect affinity on the pod.
+
+    Args:
+        vm: The VM that was migrated.
+        expected_type: The affinity type expected on the pod ("podAffinity" or "podAntiAffinity").
+    """
+    pod_affinity = vm.vmi.virt_launcher_pod.instance.to_dict()["spec"].get("affinity")
+    stale_type = POD_ANTI_AFFINITY_TYPE if expected_type == POD_AFFINITY_TYPE else POD_AFFINITY_TYPE
+    assert expected_type in pod_affinity, (
+        f"POD ({vm.vmi.virt_launcher_pod.name}) missing {expected_type}: {pod_affinity}"
+    )
+    if stale_type in pod_affinity:
+        has_stuntime_rules = any(
+            expr.get("key", "").startswith("stuntime.")
+            for rule in pod_affinity[stale_type].get("requiredDuringSchedulingIgnoredDuringExecution", [])
+            for expr in rule.get("labelSelector", {}).get("matchExpressions", [])
+        )
+        assert not has_stuntime_rules, (
+            f"POD ({vm.vmi.virt_launcher_pod.name}) has stale stuntime {stale_type}: {pod_affinity[stale_type]}"
+        )

tests/network/localnet/migration_stuntime/test_migration_stuntime.py

@@ -21,10 +21,14 @@
 from libs.vm.affinity import new_pod_affinity, new_pod_anti_affinity
 from tests.network.libs.stuntime import (
     CLIENT_VM_LABEL,
+    POD_AFFINITY_TYPE,
+    POD_ANTI_AFFINITY_TYPE,
     SERVER_VM_LABEL,
     STUNTIME_THRESHOLD_SECONDS,
+    assert_affinity_after_migration,
     measure_stuntime,
 )
+from utilities.jira import is_jira_open
 from utilities.virt import migrate_vm_and_verify
 
 pytestmark = [pytest.mark.tier3]
@@ -76,7 +80,10 @@ def test_client_migrates_off_server_node(self, admin_client, ip_family, localnet
             - Measured stuntime does not exceed the global threshold.
         """
         localnet_stuntime_client_vm.set_template_affinity(affinity=new_pod_anti_affinity(label=SERVER_VM_LABEL))
+        localnet_stuntime_client_vm.wait_for_vmi_affinity()
         migrate_vm_and_verify(vm=localnet_stuntime_client_vm, client=admin_client)
+        if is_jira_open(jira_id="CNV-90576"):
+            assert_affinity_after_migration(vm=localnet_stuntime_client_vm, expected_type=POD_ANTI_AFFINITY_TYPE)
         measured_stuntime = measure_stuntime(active_ping=active_ping)
         assert measured_stuntime <= STUNTIME_THRESHOLD_SECONDS, (
             f"Stuntime {measured_stuntime}s exceeds threshold ({STUNTIME_THRESHOLD_SECONDS}s)"
@@ -133,7 +140,10 @@ def test_client_migrates_to_server_node(self, admin_client, ip_family, localnet_
             - Measured stuntime does not exceed the global threshold.
         """
         localnet_stuntime_client_vm.set_template_affinity(affinity=new_pod_affinity(label=SERVER_VM_LABEL))
+        localnet_stuntime_client_vm.wait_for_vmi_affinity()
         migrate_vm_and_verify(vm=localnet_stuntime_client_vm, client=admin_client)
+        if is_jira_open(jira_id="CNV-90576"):
+            assert_affinity_after_migration(vm=localnet_stuntime_client_vm, expected_type=POD_AFFINITY_TYPE)
         measured_stuntime = measure_stuntime(active_ping=active_ping)
         assert measured_stuntime <= STUNTIME_THRESHOLD_SECONDS, (
             f"Stuntime {measured_stuntime}s exceeds threshold ({STUNTIME_THRESHOLD_SECONDS}s)"
@@ -161,7 +171,10 @@ def test_server_migrates_off_client_node(self, admin_client, ip_family, localnet
             - Measured stuntime does not exceed the global threshold.
         """
         localnet_stuntime_server_vm.set_template_affinity(affinity=new_pod_anti_affinity(label=CLIENT_VM_LABEL))
+        localnet_stuntime_server_vm.wait_for_vmi_affinity()
         migrate_vm_and_verify(vm=localnet_stuntime_server_vm, client=admin_client)
+        if is_jira_open(jira_id="CNV-90576"):
+            assert_affinity_after_migration(vm=localnet_stuntime_server_vm, expected_type=POD_ANTI_AFFINITY_TYPE)
         measured_stuntime = measure_stuntime(active_ping=active_ping)
         assert measured_stuntime <= STUNTIME_THRESHOLD_SECONDS, (
             f"Stuntime {measured_stuntime}s exceeds threshold ({STUNTIME_THRESHOLD_SECONDS}s)"
@@ -218,7 +231,10 @@ def test_server_migrates_to_client_node(self, admin_client, ip_family, localnet_
             - Measured stuntime does not exceed the global threshold.
         """
         localnet_stuntime_server_vm.set_template_affinity(affinity=new_pod_affinity(label=CLIENT_VM_LABEL))
+        localnet_stuntime_server_vm.wait_for_vmi_affinity()
         migrate_vm_and_verify(vm=localnet_stuntime_server_vm, client=admin_client)
+        if is_jira_open(jira_id="CNV-90576"):
+            assert_affinity_after_migration(vm=localnet_stuntime_server_vm, expected_type=POD_AFFINITY_TYPE)
         measured_stuntime = measure_stuntime(active_ping=active_ping)
         assert measured_stuntime <= STUNTIME_THRESHOLD_SECONDS, (
             f"Stuntime {measured_stuntime}s exceeds threshold ({STUNTIME_THRESHOLD_SECONDS}s)"