Re-enable extension policy tests in daily runbook (#3607)

* Enable ext_policy tests

* Address copilot comments

---------

Co-authored-by: Norberto Arrieta <narrieta@users.noreply.github.com>

Author: maddieford

tests_e2e/orchestrator/lib/agent_test_loader.py

@@ -80,6 +80,10 @@ class VmImageInfo(object):
     locations: Dict[str, List[str]]
     # Indicates that the image is available only for those VM sizes. If empty, the image should be available for all VM sizes
     vm_sizes: List[str]
+    # Optional security type (e.g. "ConfidentialVM") to use when deploying this image. When set, the deployment
+    # is forced to use this security type both for VM (via LISA's Security_Profile requirement) and for VMSS
+    # (via the 'securityType' parameter in the ARM template). When empty, the default deployment behavior is used.
+    security_type: str
 
     def __str__(self):
         return self.urn
@@ -373,12 +377,16 @@ def _load_images() -> Dict[str, List[VmImageInfo]]:
                 i.urn = description
                 i.locations = {}
                 i.vm_sizes = []
+                i.security_type = ""
             else:
                 if "urn" not in description:
                     raise Exception(f"Image {name} is missing the 'urn' property: {description}")
                 i.urn = description["urn"]
                 i.locations = description["locations"] if "locations" in description else {}
                 i.vm_sizes = description["vm_sizes"] if "vm_sizes" in description else []
+                i.security_type = description["security_type"] if "security_type" in description else ""
+                if i.security_type not in ("", "ConfidentialVM"):
+                    raise Exception(f"Invalid security_type {i.security_type} for image {name} in images.yml; expected one of '', 'ConfidentialVM'")
                 for cloud in i.locations.keys():
                     if cloud not in ["AzureCloud", "AzureChinaCloud", "AzureUSGovernment"]:
                         raise Exception(f"Invalid cloud {cloud} for image {name} in images.yml")

tests_e2e/orchestrator/lib/agent_test_suite.py

@@ -145,6 +145,9 @@ def __init__(self, metadata: TestSuiteMetadata) -> None:
         self._location: str  # Azure location (region) where test VMs are located
         self._image: str   # Image used to create the test VMs; it can be empty if LISA chose the size, or when using an existing VM
 
+        self._vm_size: str  # VM size to use when creating scale sets; empty means use the template default
+        self._security_type: str  # ARM security type (e.g. 'ConfidentialVM') to use when deploying scale sets; empty means template default
+
         self._is_vhd: bool  # True when the test VMs were created by LISA from a VHD; this is usually used to validate a new VHD and the test Agent is not installed
 
         # username and public SSH key for the admin account used to connect to the test VMs
@@ -203,6 +206,8 @@ def _initialize(self, environment: Environment, variables: Dict[str, Any], lisa_
         self._subscription_id = variables["subscription_id"]
         self._location = variables["c_location"]
         self._image = variables["c_image"]
+        self._vm_size = variables["c_vm_size"]
+        self._security_type = variables["c_security_type"]
 
         self._is_vhd = variables["c_is_vhd"]
 
@@ -903,7 +908,7 @@ def read_file(path: str) -> str:
         if self._allow_ssh != '':
             network_security_rule.add_allow_ssh_rule(self._allow_ssh)
 
-        return template, {
+        parameters = {
             "username": {"value": self._user},
             "sshPublicKey": {"value": read_file(f"{self._identity_file}.pub")},
             "vmName": {"value": scale_set_name},
@@ -913,5 +918,17 @@ def read_file(path: str) -> str:
             "version": {"value": version}
         }
 
+        # If the image definition (in images.yml) or the runbook specifies a VM size, use it; otherwise fall back
+        # to the template default.
+        if self._vm_size != '':
+            parameters["vmSize"] = {"value": self._vm_size}
+
+        # If the image definition (in images.yml) declares a security type (e.g. 'ConfidentialVM'), set it on the
+        # scale set; otherwise the template default ('Standard') is used.
+        if self._security_type != '':
+            parameters["securityType"] = {"value": self._security_type}
+
+        return template, parameters
+
 
 

tests_e2e/orchestrator/lib/agent_test_suite_combinator.py

@@ -204,6 +204,7 @@ def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]
                     continue
 
                 vm_size = self._get_vm_size(image)
+                security_type = image.security_type
 
                 locations: List[str] = self._get_locations(test_suite_info, image)
                 if len(locations) == 0:
@@ -223,6 +224,7 @@ def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]
                                 marketplace_image=marketplace_image,
                                 location=location,
                                 vm_size=vm_size,
+                                security_type=security_type,
                                 test_suite_info=test_suite_info)
                         else:
                             env = self.create_vm_environment(
@@ -232,6 +234,7 @@ def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]
                                 shared_gallery=shared_gallery,
                                 location=location,
                                 vm_size=vm_size,
+                                security_type=security_type,
                                 test_suite_info=test_suite_info)
                         environments.append(env)
                     else:
@@ -247,6 +250,7 @@ def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]
                                     marketplace_image=marketplace_image,
                                     location=location,
                                     vm_size=vm_size,
+                                    security_type=security_type,
                                     test_suite_info=test_suite_info)
                             else:
                                 env = self.create_vm_environment(
@@ -256,6 +260,7 @@ def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]
                                     shared_gallery=shared_gallery,
                                     location=location,
                                     vm_size=vm_size,
+                                    security_type=security_type,
                                     test_suite_info=test_suite_info)
                             shared_environments[env_name] = env
 
@@ -371,7 +376,7 @@ def create_existing_vmss_environment(self, test_suites: List[str]) -> Dict[str,
             "c_test_suites": loader.test_suites,
         }
 
-    def create_vm_environment(self, env_name: str, marketplace_image: str, vhd: str, shared_gallery: str, location: str, vm_size: str, test_suite_info: TestSuiteInfo) -> Dict[str, Any]:
+    def create_vm_environment(self, env_name: str, marketplace_image: str, vhd: str, shared_gallery: str, location: str, vm_size: str, test_suite_info: TestSuiteInfo, security_type: str = "") -> Dict[str, Any]:
         #
         # Custom ARM templates (to create the test VMs) require special handling. These templates are processed by the azure_update_arm_template
         # hook, which does not have access to the runbook variables. Instead, we use a dummy VM tag named "templates" and pass the
@@ -435,9 +440,24 @@ def create_vm_environment(self, env_name: str, marketplace_image: str, vhd: str,
                     }
                 ]
             }
+        elif security_type == "ConfidentialVM":
+            # On the VM path LISA performs the deployment, so the security type must be expressed as a LISA feature
+            # requirement on 'c_platform' (LISA does not look at the 'c_security_type' variable, which is consumed
+            # only by 'AgentTestSuite' on the VMSS path). This forces LISA to deploy the image as a Confidential VM
+            # regardless of which security profiles the image and VM size happen to support; without it, LISA's
+            # priority-based selection may pick a non-CVM profile. Note that LISA's SecurityProfileType enum uses
+            # the lowercase value 'cvm' (which it maps internally to ARM's 'ConfidentialVM').
+            environment['c_platform'][0]['requirement']["features"] = {
+                "items": [
+                    {
+                        "type": "Security_Profile",
+                        "security_profile": "cvm"
+                    }
+                ]
+            }
         return environment
 
-    def create_vmss_environment(self, env_name: str, marketplace_image: str, location: str, vm_size: str, test_suite_info: TestSuiteInfo) -> Dict[str, Any]:
+    def create_vmss_environment(self, env_name: str, marketplace_image: str, location: str, vm_size: str, test_suite_info: TestSuiteInfo, security_type: str = "") -> Dict[str, Any]:
         return {
             "c_platform": [
                 {
@@ -461,6 +481,9 @@ def create_vmss_environment(self, env_name: str, marketplace_image: str, locatio
             "c_location": location,
             "c_image": marketplace_image,
             "c_is_vhd": False,
+            # On the VMSS path the scale set is deployed by 'AgentTestSuite' using our own ARM template
+            # (vmss.json), bypassing LISA.
+            "c_security_type": security_type,
             "c_vm_size": vm_size,
             "vm_tags": {}
         }
@@ -484,6 +507,7 @@ def _get_runbook_images(self, loader: AgentTestLoader) -> List[VmImageInfo]:
         i.urn = self.runbook.image  # Note that this could be a URN or the URI for a VHD, or an image from a shared gallery
         i.locations = []
         i.vm_sizes = []
+        i.security_type = ""
 
         return [i]
 
@@ -503,6 +527,7 @@ def _get_test_suite_images(suite: TestSuiteInfo, loader: AgentTestLoader) -> Lis
                     i.urn = image
                     i.locations = []
                     i.vm_sizes = []
+                    i.security_type = ""
                     image_list = [i]
                 else:
                     image_list = loader.images[image]

tests_e2e/orchestrator/runbook.yml

@@ -45,11 +45,8 @@ variable:
       - agent_status
       - agent_update
       - ext_cgroups
-#
-# TODO: These tests are disabled temporarily since our test account does not have quota to create the Confidential VMs required by the tests.
-#
-#      - ext_policy
-#      - ext_policy_with_dependencies
+      - ext_policy
+      - ext_policy_with_dependencies
       - ext_sequencing
       - ext_signature_validation
       - ext_telemetry_pipeline
@@ -221,6 +218,15 @@ variable:
     value: false
     is_case_visible: true
 
+  #
+  # Security type to use when deploying the VM/VMSS resource (e.g. "ConfidentialVM"). Empty means use the default
+  # deployment behavior. Populated by the AgentTestSuiteCombinator from the 'security_type' property on the
+  # image in images.yml.
+  #
+  - name: c_security_type
+    value: ""
+    is_case_visible: true
+
 environment: $(c_environment)
 
 platform: $(c_platform)

tests_e2e/orchestrator/templates/vmss.json

@@ -26,6 +26,18 @@
         },
         "version": {
             "type": "string"
+        },
+        "vmSize": {
+            "type": "string",
+            "defaultValue": "Standard_D2s_v3"
+        },
+        "securityType": {
+            "type": "string",
+            "defaultValue": "Standard",
+            "allowedValues": [
+                "Standard",
+                "ConfidentialVM"
+            ]
         }
     },
     "variables": {
@@ -167,7 +179,7 @@
                 "[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]"
             ],
             "sku": {
-                "name": "Standard_D2s_v3",
+                "name": "[parameters('vmSize')]",
                 "tier": "Standard",
                 "capacity": 3
             },
@@ -199,7 +211,8 @@
                             "createOption": "FromImage",
                             "caching": "ReadWrite",
                             "managedDisk": {
-                                "storageAccountType": "Premium_LRS"
+                                "storageAccountType": "Premium_LRS",
+                                "securityProfile": "[if(equals(parameters('securityType'), 'ConfidentialVM'), createObject('securityEncryptionType', 'DiskWithVMGuestState'), json('null'))]"
                             },
                             "diskSizeGB": 64
                         },
@@ -210,6 +223,7 @@
                             "version": "[parameters('version')]"
                         }
                     },
+                    "securityProfile": "[if(equals(parameters('securityType'), 'ConfidentialVM'), createObject('securityType', 'ConfidentialVM', 'uefiSettings', createObject('secureBootEnabled', true(), 'vTpmEnabled', true())), json('null'))]",
                     "diagnosticsProfile": {
                         "bootDiagnostics": {
                             "enabled": true

tests_e2e/test_suites/ext_policy.yml

@@ -1,15 +1,13 @@
 #
 # The test suite verifies that disallowed extensions are not processed, but the agent should still report status.
 #
+# TODO: This test suite takes ~30 minutes to run. This should be optimized to reduce impact to pipeline run times.
 name: "ExtensionPolicy"
 tests:
   - "ext_policy/ext_policy.py"
 images:
-  - "endorsed"
+  - "random(endorsed,10)" # TODO: Remove randomization and run on all endorsed images once the test suite is optimized to reduce runtime.
   - "cvm-endorsed"
-# This test is executed in southcentralus as a workaround for recurring fabric "ServiceUnavailableFault" issues observed in westus2.
-locations: "AzureCloud:southcentralus"
-# TODO: This test is currently failing on usgov cloud due to an issue with the GuestConfig extension. Re-enable once the extension fix has been rolled out.
-skip_on_clouds:
-  - "AzureUSGovernment"
 owns_vm: false
+skip_on_images:
+  - "AzureChinaCloud:debian_11" # The ConfigurationforLinux-1.26.109 extension is failing on Debian 11 in China cloud only; skip this image until the issue in the extension is fixed

tests_e2e/test_suites/ext_policy_with_dependencies.yml

@@ -4,11 +4,11 @@
 name: "ExtPolicyWithDependencies"
 tests:
   - "ext_policy/ext_policy_with_dependencies.py"
-images: "endorsed"
+images:
+  - "endorsed"
+  - "cvm-endorsed"
 executes_on_scale_set: true
 owns_vm: false
-# This test is executed in southcentralus as a workaround for recurring fabric "ServiceUnavailableFault" issues observed in westus2.
-locations: "AzureCloud:southcentralus"
 
 # TODO: Currently AlmaLinux is not available for scale sets; enable this image when it is available.
 skip_on_images:
@@ -18,4 +18,4 @@ skip_on_images:
 
 # TODO: The current deployment of VmAccess 1.5.22 prevents the extension from uninstalling; enable this test when the issue is fixed
 skip_on_clouds:
-  - "AzureUSGovernment"
+  - "AzureUSGovernment"

tests_e2e/test_suites/ext_signature_validation.yml

@@ -7,7 +7,6 @@ tests:
 # Extension signature is sent by CRP only for CVMs, so this test suite should run exclusively on CVMs.
 images: "cvm-endorsed"
 # Extension signatures are currently only available in the public cloud, so we skip this test on other clouds.
-locations: "AzureCloud:westeurope"
 skip_on_clouds:
   - "AzureChinaCloud"
   - "AzureUSGovernment"