Skip tests requiring a scale set when using a VHD (#3582)

Author: narrieta

tests_e2e/orchestrator/lib/agent_test_result.py

@@ -20,14 +20,29 @@
 
 # Disable those warnings, since 'lisa' is an external, non-standard, dependency
 #     E0401: Unable to import 'lisa' (import-error)
-#     etc
-from lisa import (  # pylint: disable=E0401
-    notifier
-)
+from lisa import notifier  # pylint: disable=E0401
 from lisa.messages import TestStatus, TestResultMessage  # pylint: disable=E0401
+
+from typing import Optional
+
 from azurelinuxagent.common.future import UTC
 
 
+class AgentTestResultMessage(TestResultMessage):
+    def __init__(self, suite_name: str, test_name: str, status: TestStatus):
+        super().__init__()
+        self.type: str = "AgentTestResultMessage"
+        self.id_: str = str(uuid.uuid4())
+        self.status: TestStatus = status
+        self.suite_full_name: str = suite_name
+        self.suite_name: str = suite_name
+        self.full_name: str = test_name
+        self.name: str = test_name
+        self.elapsed: float = 0
+        self.message: str = ""
+        self.stacktrace: Optional[str] = None
+
+
 class AgentTestResult:
     @staticmethod
     def report(
@@ -42,15 +57,7 @@ def report(
         Reports a test result to the junit notifier
         """
         # The junit notifier requires an initial RUNNING message in order to register the test in its internal cache.
-        msg: TestResultMessage = TestResultMessage()
-        msg.type = "AgentTestResultMessage"
-        msg.id_ = str(uuid.uuid4())
-        msg.status = TestStatus.RUNNING
-        msg.suite_full_name = suite_name
-        msg.suite_name = msg.suite_full_name
-        msg.full_name = test_name
-        msg.name = msg.full_name
-        msg.elapsed = 0
+        msg: AgentTestResultMessage = AgentTestResultMessage(suite_name, test_name, TestStatus.RUNNING)
 
         notifier.notify(msg)
 

tests_e2e/orchestrator/lib/agent_test_suite_combinator.py

@@ -154,6 +154,8 @@ def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]
 
         runbook_images = self._get_runbook_images(loader)
 
+        all_suites_require_vmss = all(s.executes_on_scale_set for s in loader.test_suites)
+
         skip_test_suites: List[str] = []
         skip_test_suites_images: List[Tuple[str, str]] = []
         for test_suite_info in loader.test_suites:
@@ -190,7 +192,16 @@ def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]
                     shared_gallery = ""
 
                 if test_suite_info.executes_on_scale_set and (vhd != "" or shared_gallery != ""):
-                    raise Exception("VHDS and images from galleries are currently not supported on scale sets.")
+                    if all_suites_require_vmss:
+                        raise Exception("VHDs and images from galleries are currently not supported on scale sets")
+                    
+                    AgentTestResult.report(
+                        f"{test_suite_info.name}-{image_name}",
+                        test_suite_info.name,
+                        TestStatus.SKIPPED,
+                        datetime.datetime.now(datetime.timezone.utc),
+                        message="VHDs and images from galleries are currently not supported on scale sets.")
+                    continue
 
                 vm_size = self._get_vm_size(image)