Merge branch 'develop' into mayansingh/add_acl_support

Author: mayankfz

azurelinuxagent/agent.py

@@ -34,6 +34,7 @@
 from azurelinuxagent.common.exception import CGroupsException
 from azurelinuxagent.ga import logcollector, cgroupconfigurator
 from azurelinuxagent.ga.cgroupcontroller import AGENT_LOG_COLLECTOR
+from azurelinuxagent.ga.confidential_vm_info import ConfidentialVMInfo
 from azurelinuxagent.ga.cpucontroller import _CpuController
 from azurelinuxagent.ga.cgroupapi import create_cgroup_api, InvalidCgroupMountpointException
 from azurelinuxagent.ga.firewall_manager import FirewallManager, IpTables
@@ -211,6 +212,12 @@ def collect_logs(self, is_full_mode):
         else:
             logger.info("Running log collector mode normal")
 
+        # Initialize the CVM info before initializing the telemetry, since the CVM info is part of the common parameters for telemetry events.
+        try:
+            ConfidentialVMInfo.fetch_and_initialize_cvm_info()
+        except Exception as ex:
+            logger.warn("Failed to get virtual machine security type from IMDS, will assume this is not a Confidential Virtual Machine: {0}".format(
+                ustr(ex)))
         LogCollector.initialize_telemetry()
 
         # Check the cgroups unit

azurelinuxagent/common/AgentGlobals.py

@@ -30,10 +30,30 @@ class AgentGlobals(object):
     #
     _container_id = GUID_ZERO
 
+    #
+    # The telemetry modules require the information about whether the agent is running in a CVM or not. This variable
+    # will be updated when the CVM info is initialized in ConfidentialVMInfo. There are three possible values:
+    #   - None
+    #   - True
+    #   - False
+    # The value is None when the CVM info has not yet been initialized.
+    #
+    _is_cvm = None
+
     @staticmethod
     def get_container_id():
         return AgentGlobals._container_id
 
     @staticmethod
     def update_container_id(container_id):
         AgentGlobals._container_id = container_id
+
+    @staticmethod
+    def get_is_cvm():
+        if AgentGlobals._is_cvm is None:
+            raise Exception("CVM info has not been initialized yet")
+        return AgentGlobals._is_cvm
+
+    @staticmethod
+    def update_is_cvm(is_cvm):
+        AgentGlobals._is_cvm = is_cvm

azurelinuxagent/common/event.py

@@ -394,8 +394,11 @@ def __init__(self):
 
         # Parameters from OS
         osutil = get_osutil()
+        # Determining IsCVM requires a network call. Set as uninitialized for now until common parameters are
+        # initialized with real values in initialize_vminfo_common_parameters()
         keyword_name = {
-            "CpuArchitecture": osutil.get_vm_arch()
+            "CpuArchitecture": osutil.get_vm_arch(),
+            "IsCVM": "IsCVM_UNINITIALIZED"
         }
         self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.OSVersion, EventLogger._get_os_version()))
         self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.ExecutionMode, AGENT_EXECUTION_MODE))
@@ -463,6 +466,25 @@ def initialize_vminfo_common_parameters(self, protocol):
         except Exception as e:
             logger.warn("Failed to get IMDS info; will be missing from telemetry: {0}", ustr(e))
 
+        # The KeywordName column is initialized with the CPUArch in EventLogger.__init__(). The security type is
+        # not yet discovered at that time because it requires a network call, so we update KeywordName here with the
+        # IsCVM value.
+        # The security type is initialized by the ConfidentialVMInfo class because it fetches metadata from IMDS with
+        # the minimum version that supports the security type field. We do not use that minimum version in the IMDS
+        # request in this method due to inadequate saturation of that version in the fleet. When the ConfidentialVMInfo
+        # class attributes are initialized, AgentGlobals is also updated with the security type, so we can get the
+        # security type in this module without introducing dependencies on the ConfidentialVMInfo class.
+        try:
+            keyword_name_str = parameters[CommonTelemetryEventSchema.KeywordName].value               # Get the current value of keywordName
+            keyword_name_json = json.loads(keyword_name_str)                                          # Convert the string to JSON
+            # AgentGlobals.get_is_cvm() raises if cvm info is not initialized so the IsCVM value in the keywordName
+            # column would remain uninitialized in that case
+            is_cvm = AgentGlobals.get_is_cvm()                                                        # Get the CVM state from AgentGlobals
+            keyword_name_json["IsCVM"] = is_cvm                                                       # Update the security type in the JSON
+            parameters[CommonTelemetryEventSchema.KeywordName].value = json.dumps(keyword_name_json)  # Convert the JSON back to string and update the value of keywordName
+        except Exception as e:
+            logger.warn("Failed to update the KeywordName column with IsCVM; will be missing from telemetry: {0}", ustr(e))
+
     def save_event(self, data):
         if self.event_dir is None:
             logger.warn("Cannot save event -- Event reporter is not initialized.")

azurelinuxagent/ga/confidential_vm_info.py

@@ -19,6 +19,7 @@
 
 import json
 
+from azurelinuxagent.common.AgentGlobals import AgentGlobals
 from azurelinuxagent.common.protocol.imds import ImdsClient
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.exception import HttpError
@@ -77,10 +78,12 @@ def fetch_and_initialize_cvm_info():
         try:
             security_type = ConfidentialVMInfo._fetch_security_type_from_imds()
             ConfidentialVMInfo._is_confidential_vm = (security_type == SecurityType.ConfidentialVM)
+            AgentGlobals.update_is_cvm(ConfidentialVMInfo._is_confidential_vm)
         except Exception as ex:
             # TODO: For now, in the case of IMDS failure, we treat the VM as non-CVM until the next agent service start.
             # This should be improved to better distinguish IMDS issues from true security type.
             ConfidentialVMInfo._is_confidential_vm = False
+            AgentGlobals.update_is_cvm(False)
             raise ex
 
     @staticmethod

azurelinuxagent/ga/update.py

@@ -357,7 +357,9 @@ def run(self, debug=False):
                 )
             logger.info(os_info_msg)
 
-            # Initialize Confidential VM info but defer sending telemetry until common parameters are initialized
+            # Initialize Confidential VM info but defer sending telemetry until common parameters are initialized.
+            # ConfidentialVMInfo.fetch_and_initialize_cvm_info() should be called to initialize AgentGlobals._is_cvm
+            # before initialize_event_logger_vminfo_common_parameters_and_protocol() is called.
             cvm_info_err = None
             try:
                 ConfidentialVMInfo.fetch_and_initialize_cvm_info()

tests/common/test_event.py

@@ -71,7 +71,7 @@ def setUp(self):
             CommonTelemetryEventSchema.EventTid: threading.current_thread().ident,
             CommonTelemetryEventSchema.EventPid: os.getpid(),
             CommonTelemetryEventSchema.TaskName: threading.current_thread().name,
-            CommonTelemetryEventSchema.KeywordName: json.dumps({"CpuArchitecture": platform.machine()}),
+            CommonTelemetryEventSchema.KeywordName: json.dumps({"CpuArchitecture": platform.machine(), "IsCVM": False}),
             # common parameters computed from the OS platform
             CommonTelemetryEventSchema.OSVersion: EventLoggerTools.get_expected_os_version(),
             CommonTelemetryEventSchema.ExecutionMode: AGENT_EXECUTION_MODE,

tests/ga/test_confidential_vm_info.py

@@ -17,6 +17,7 @@
 
 import os
 
+from azurelinuxagent.common.AgentGlobals import AgentGlobals
 from azurelinuxagent.ga.confidential_vm_info import ConfidentialVMInfo
 from tests.lib.tools import AgentTestCase, MagicMock, patch, data_dir
 
@@ -39,13 +40,17 @@ def test_should_identify_confidential_vm(self):
             ConfidentialVMInfo.fetch_and_initialize_cvm_info()
             is_cvm = ConfidentialVMInfo.is_confidential_vm()
             self.assertTrue(is_cvm)
+            self.assertTrue(AgentGlobals.get_is_cvm())      # Verify that AgentGlobals was also updated
 
     def test_should_identify_non_confidential_vm(self):
         with patch('azurelinuxagent.ga.confidential_vm_info.ImdsClient.get_metadata') as mock_get_metadata:
             self._setup_mock_imds_from_file(mock_get_metadata, os.path.join(data_dir, "imds", "trusted_vm_metadata.json"))
             ConfidentialVMInfo.fetch_and_initialize_cvm_info()
             is_cvm = ConfidentialVMInfo.is_confidential_vm()
             self.assertFalse(is_cvm)
+            # Verify that AgentGlobals was also updated
+            self.assertFalse(AgentGlobals.get_is_cvm())
+            self.assertIsNotNone(AgentGlobals.get_is_cvm())
 
     def test_should_return_false_when_imds_unavailable(self):
         with patch('azurelinuxagent.ga.confidential_vm_info.ImdsClient.get_metadata') as mock_get_metadata:
@@ -61,6 +66,9 @@ def test_should_return_false_when_imds_unavailable(self):
             # After exception, is_confidential_vm should be False
             is_cvm = ConfidentialVMInfo.is_confidential_vm()
             self.assertFalse(is_cvm)
+            # Verify that AgentGlobals was also updated
+            self.assertFalse(AgentGlobals.get_is_cvm())
+            self.assertIsNotNone(AgentGlobals.get_is_cvm())
 
     def test_should_always_return_false_after_transient_imds_failure(self):
         with patch('azurelinuxagent.ga.confidential_vm_info.ImdsClient.get_metadata') as mock_get_metadata:
@@ -78,8 +86,14 @@ def test_should_always_return_false_after_transient_imds_failure(self):
                 ConfidentialVMInfo.fetch_and_initialize_cvm_info()
             first_call = ConfidentialVMInfo.is_confidential_vm()
             self.assertFalse(first_call)
+            # Verify that AgentGlobals was also updated
+            self.assertFalse(AgentGlobals.get_is_cvm())
+            self.assertIsNotNone(AgentGlobals.get_is_cvm())
             second_call = ConfidentialVMInfo.is_confidential_vm()
             self.assertFalse(second_call)
+            # Verify that AgentGlobals was also updated
+            self.assertFalse(AgentGlobals.get_is_cvm())
+            self.assertIsNotNone(AgentGlobals.get_is_cvm())
 
             # Verify IMDS was only called once
             self.assertEqual(mock_get_metadata.call_count, 1)

tests/ga/test_send_telemetry_events.py

@@ -77,6 +77,8 @@ def http_post_handler(url, body, **__):
             protocol_util.get_protocol = Mock(return_value=protocol)
             send_telemetry_events_handler = get_send_telemetry_events_handler(protocol_util)
             send_telemetry_events_handler.event_calls = []
+            ConfidentialVMInfo = MagicMock()
+            ConfidentialVMInfo.is_confidential_vm = Mock(return_vale=False)
             with patch("azurelinuxagent.ga.send_telemetry_events.SendTelemetryEventsHandler._MIN_EVENTS_TO_BATCH",
                        batching_queue_limit):
                 with patch("azurelinuxagent.ga.send_telemetry_events.SendTelemetryEventsHandler._MAX_TIMEOUT", timeout):
@@ -383,7 +385,7 @@ def test_it_should_enqueue_and_send_events_properly(self, mock_lib_dir, *_):
                              '<Param Name="ImageOrigin" Value="2468" T="mt:uint64" />' \
                              ']]></Event>'.format(AGENT_VERSION, TestSendTelemetryEventsHandler._TEST_EVENT_OPERATION, CURRENT_AGENT, test_opcodename, test_eventtid,
                                                   test_eventpid, test_taskname, osversion, int(osutil.get_total_mem()),
-                                                  osutil.get_processor_cores(), json.dumps({"CpuArchitecture": platform.machine()})).encode('utf-8')
+                                                  osutil.get_processor_cores(), json.dumps({"CpuArchitecture": platform.machine(), "IsCVM": False})).encode('utf-8')
 
             self.assertIn(sample_message, collected_event)
 

tests/lib/event_logger_tools.py

@@ -55,7 +55,8 @@ def initialize_event_logger(event_dir):
 
         with mock_wire_protocol(wire_protocol_data.DATA_FILE) as mock_protocol:
             with tools.patch("azurelinuxagent.common.event.get_imds_client", return_value=mock_imds_client):
-                event.initialize_event_logger_vminfo_common_parameters_and_protocol(mock_protocol)
+                with tools.patch("azurelinuxagent.common.AgentGlobals.AgentGlobals.get_is_cvm", return_value=False):
+                    event.initialize_event_logger_vminfo_common_parameters_and_protocol(mock_protocol)
 
     @staticmethod
     def get_expected_os_version():

tests_e2e/orchestrator/lib/agent_test_result.py

@@ -20,14 +20,29 @@
 
 # Disable those warnings, since 'lisa' is an external, non-standard, dependency
 #     E0401: Unable to import 'lisa' (import-error)
-#     etc
-from lisa import (  # pylint: disable=E0401
-    notifier
-)
+from lisa import notifier  # pylint: disable=E0401
 from lisa.messages import TestStatus, TestResultMessage  # pylint: disable=E0401
+
+from typing import Optional
+
 from azurelinuxagent.common.future import UTC
 
 
+class AgentTestResultMessage(TestResultMessage):
+    def __init__(self, suite_name: str, test_name: str, status: TestStatus):
+        super().__init__()
+        self.type: str = "AgentTestResultMessage"
+        self.id_: str = str(uuid.uuid4())
+        self.status: TestStatus = status
+        self.suite_full_name: str = suite_name
+        self.suite_name: str = suite_name
+        self.full_name: str = test_name
+        self.name: str = test_name
+        self.elapsed: float = 0
+        self.message: str = ""
+        self.stacktrace: Optional[str] = None
+
+
 class AgentTestResult:
     @staticmethod
     def report(
@@ -42,15 +57,7 @@ def report(
         Reports a test result to the junit notifier
         """
         # The junit notifier requires an initial RUNNING message in order to register the test in its internal cache.
-        msg: TestResultMessage = TestResultMessage()
-        msg.type = "AgentTestResultMessage"
-        msg.id_ = str(uuid.uuid4())
-        msg.status = TestStatus.RUNNING
-        msg.suite_full_name = suite_name
-        msg.suite_name = msg.suite_full_name
-        msg.full_name = test_name
-        msg.name = msg.full_name
-        msg.elapsed = 0
+        msg: AgentTestResultMessage = AgentTestResultMessage(suite_name, test_name, TestStatus.RUNNING)
 
         notifier.notify(msg)