remove memory limit (#3604) (#3606)

(cherry picked from commit 9217828)
(cherry picked from commit 93d7517)

Author: nagworld9

azurelinuxagent/common/event.py

@@ -137,6 +137,7 @@ class WALAEventOperation:
     RequestedStateDisabled = "RequestedStateDisabled"
     RequestedVersionMismatch = "RequestedVersionMismatch"
     ResetFirewall = "ResetFirewall"
+    ResetMemory = "ResetMemory"
     Restart = "Restart"
     SetCGroupsLimits = "SetCGroupsLimits"
     SignatureValidation = "SignatureValidation"                         # Event for general logs related to package signature or manifest validation that don't fall under a specific operation.

azurelinuxagent/ga/cgroupconfigurator.py

@@ -152,6 +152,8 @@ def initialize(self):
                 # Before agent setup, cleanup the old agent setup (drop-in files) since new agent uses different approach(systemctl) to setup cgroups.
                 log_cgroup_info("Cleaning up old agent setup (drop-in files), if any")
                 self._cleanup_old_agent_setup()
+                if self.using_cgroup_v2():
+                    self._reset_resource_quota(systemd.get_agent_unit_name(), ResourceName.MEMORY, ignore_enforce_check=True)
 
                 # Notes about slice setup:
                 #   For machines where daemon version did not already create azure.slice, the
@@ -183,8 +185,7 @@ def initialize(self):
                     if isinstance(controller, _CpuController) and self._cgroups_api.can_enforce_cpu():
                         self._set_resource_quota(agent_unit_name, {ResourceName.CPU:conf.get_agent_cpu_quota()})
                         controller.track_throttle_time(True)  # CPU controller track the throttle time only when CPU quota is set
-                    elif isinstance(controller, _MemoryController) and self._cgroups_api.can_enforce_memory():
-                        self._set_resource_quota(agent_unit_name, {ResourceName.MEMORY:conf.get_agent_memory_quota()})
+                    elif isinstance(controller, _MemoryController):
                         self._agent_memory_metrics = controller
                     CGroupsTelemetry.track_cgroup_controller(controller)
 
@@ -194,9 +195,9 @@ def initialize(self):
                 log_cgroup_info('Agent cgroups enabled: {0}'.format(self._agent_cgroups_enabled))
                 self._initialized = True
 
-                if self._cgroups_api is not None and not self._cgroups_api.can_enforce_cpu():
-                    # If agent cgroups are not enabled or quotas not enabled, reset the quota for the agent unit
-                    log_cgroup_info("Reset CPU quota if agent cgroups were not enabled for enforcement")
+                # If agent cgroups are not enabled or CPU quotas cannot be enforced, reset the quota for the agent unit
+                if self._cgroups_api is not None and (not self._agent_cgroups_enabled or not self._cgroups_api.can_enforce_cpu()):
+                    log_cgroup_info("Reset CPU quota since agent cgroups are not enabled for enforcement")
                     self._reset_resource_quota(systemd.get_agent_unit_name(), ResourceName.CPU, ignore_enforce_check=True)
 
         def _check_cgroups_supported(self):

azurelinuxagent/ga/update.py

@@ -332,6 +332,36 @@ def run(self, debug=False):
         """
 
         try:
+            reset_memory_telemetry = []
+            msg1 = "[{0}] Starting ext_handler process".format(datetime.now(UTC))
+            logger.info(msg1)
+            reset_memory_telemetry.append(msg1)
+            try:
+                if systemd.is_systemd():
+                    unit_name = systemd.get_agent_unit_name()
+                    current_limit = systemd.get_unit_property(unit_name, "MemoryHigh").strip().lower()  # check if the property exists
+                    if current_limit != "infinity":
+                        systemd.set_unit_run_time_property(unit_name, "MemoryHigh", "")
+                        msg2 = "[{0}] Reset agent cgroup MemoryHigh property to infinity finished.".format(datetime.now(UTC))
+                        logger.info(msg2)
+                        reset_memory_telemetry.append(msg2)
+                        new_limit = systemd.get_unit_property(unit_name, "MemoryHigh")
+                        msg3 = "[{0}] Current MemoryHigh is {1}.".format(datetime.now(UTC), new_limit)
+                        logger.info(msg3)
+                        reset_memory_telemetry.append(msg3)
+                    else:
+                        msg4 = "[{0}] Agent cgroup MemoryHigh property is already set to infinity, no need to reset it.".format(datetime.now(UTC))
+                        logger.info(msg4)
+                        reset_memory_telemetry.append(msg4)
+                else:
+                    msg6 = "[{0}] Systemd is not present, skipping reset of agent cgroup MemoryHigh property.".format(datetime.now(UTC))
+                    logger.info(msg6)
+                    reset_memory_telemetry.append(msg6)
+            except Exception as e:
+                msg5 = "[{0}] Failed to reset agent cgroup MemoryHigh property: {1}".format(datetime.now(UTC), ustr(e))
+                logger.info(msg5)
+                reset_memory_telemetry.append(msg5)
+
             logger.info("{0} (Goal State Agent version {1})", AGENT_LONG_NAME, AGENT_VERSION)
             logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION)
             logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO)
@@ -377,6 +407,9 @@ def run(self, debug=False):
             # Initialize the common parameters for telemetry events
             initialize_event_logger_vminfo_common_parameters_and_protocol(protocol)
 
+            # reporting reset memory telemetry
+            add_event(AGENT_NAME, op=WALAEventOperation.ResetMemory, message="\n".join(reset_memory_telemetry))
+
             # Send telemetry if protocol endpoint is not the known WireServer endpoint.
             endpoint = protocol.get_endpoint()
             if endpoint is not None and endpoint != KNOWN_WIRESERVER_IP:

tests/ga/test_cgroupconfigurator.py

@@ -226,12 +226,45 @@ def test_initialize_should_start_tracking_other_controllers_when_one_is_not_pres
                     any(cg for cg in tracked if tracked[cg].name == AGENT_NAME_TELEMETRY and 'cpu' in cg),
                     "The Agent's cpu is being tracked. Tracked: {0}".format(tracked))
 
-    def test_agent_enforcement_enabled_in_v2(self):
+    def test_agent_should_set_cpu_quota_and_not_set_memory_quota_in_v2(self):
         with self._get_cgroup_configurator_v2() as configurator:
             cmd1 = 'systemctl set-property walinuxagent.service CPUQuota=50% --runtime'
-            cmd2 = 'systemctl set-property walinuxagent.service MemoryHigh=314572800 --runtime'
             self.assertIn(cmd1, configurator.mocks.commands_call_list, "The command to set CPU quota was not called")
-            self.assertIn(cmd2, configurator.mocks.commands_call_list, "The command to set Memory quota was not called")
+            # The agent no longer enforces a memory limit via systemd; it must never set MemoryHigh to a value.
+            cmd2 = "systemctl set-property walinuxagent.service MemoryHigh=314572800 --runtime"
+            self.assertNotIn(cmd2, configurator.mocks.commands_call_list, "The command to set Memory quota was called")
+
+    def test_agent_should_reset_memory_quota_in_v2_when_previously_set(self):
+        # Simulate a previously-set MemoryHigh on the agent unit; the agent should reset it on initialize.
+        command_mocks = [MockCommand(r"^systemctl show (.+) --property MemoryHigh$",
+                                     '''MemoryHigh=314572800
+                                     ''')]
+        with self._get_cgroup_configurator_v2(mock_commands=command_mocks) as configurator:
+            cmd = 'systemctl set-property walinuxagent.service MemoryHigh= --runtime'
+            self.assertIn(cmd, configurator.mocks.commands_call_list,
+                          "The command to reset the Memory quota was not called")
+
+    def test_agent_should_not_reset_memory_quota_in_v2_when_already_infinity(self):
+        # The default v2 mock returns MemoryHigh=infinity; the reset must be a no-op (no systemctl call).
+        with self._get_cgroup_configurator_v2() as configurator:
+            for cmd in configurator.mocks.commands_call_list:
+                self.assertNotIn(
+                    "systemctl set-property walinuxagent.service MemoryHigh= --runtime", cmd,
+                    "MemoryHigh reset should not be issued when current value is already infinity. Command: {0}".format(cmd))
+
+    def test_agent_should_reset_cpu_quota_when_previously_set_and_agent_not_enabled_now(self):
+        command_mocks = [
+                 MockCommand(r"^systemctl show walinuxagent\.service --property ControlGroup$",
+                             '''ControlGroup=/azure.slice/walinuxagent.service
+                             '''),
+                 MockCommand(r"^systemctl show (.+) --property CPUQuotaPerSecUSec$",
+                             '''CPUQuotaPerSecUSec=5ms
+                             ''')
+                         ]
+        with self._get_cgroup_configurator_v2(mock_commands=command_mocks) as configurator:
+            cmd = 'systemctl set-property walinuxagent.service CPUQuota= --runtime'
+            self.assertIn(cmd, configurator.mocks.commands_call_list,
+                          "The command to reset the cpu quota was not called")
 
     def test_accounting_properties_not_set_explicitly_in_cgroupv2(self):
         with self._get_cgroup_configurator_v2() as configurator:

tests_e2e/tests/scripts/agent_memory_quota-check_agent_memory_quota.py

@@ -45,8 +45,8 @@ def prepare_agent():
     log.info("Executing script update-waagent-conf to enable agent cgroups config flag")
     result = shellutil.run_command(["update-waagent-conf", "Debug.CgroupCheckPeriod=30", "Debug.CgroupLogMetrics=y",
                                     "Debug.CgroupDisableOnProcessCheckFailure=n",
-                                    "Debug.CgroupDisableOnQuotaCheckFailure=n",
-                                    "Debug.AgentMemoryQuota=104857600"])
+                                    "Debug.CgroupDisableOnQuotaCheckFailure=n"
+                                    ])
     log.info("Successfully enabled agent cgroups config flag: {0}".format(result))
 
     found: bool = retry_if_false(
@@ -55,24 +55,24 @@ def prepare_agent():
         fail("Agent cgroups not enabled")
 
 
-def verify_agent_has_memory_quota_set():
+def verify_agent_has_no_memory_quota_set():
     """
-    This method verifies that the agent's cgroup has memory quota set
+    This method verifies that the agent's cgroup does NOT have a memory quota set.
+    The agent no longer enforces a memory limit via systemd; MemoryHigh should be 'infinity'.
     """
-    log.info("** Verifying agent cgroup has memory quota set")
+    log.info("** Verifying agent cgroup has no memory quota set (MemoryHigh=infinity)")
 
-    def check_memory_quota() -> bool:
+    def check_no_memory_quota() -> bool:
         quota = get_agent_memory_quota()
-        if quota is None or quota == "infinity":
-            return False
-        return True
+        # 'infinity' means no memory limit is enforced.
+        return quota.strip().lower() == "infinity"
 
-    found: bool = retry_if_false(check_memory_quota)
+    found: bool = retry_if_false(check_no_memory_quota)
     if found:
         log.info("Agent Memory Quota: %s", get_agent_memory_quota())
-        log.info("Successfully verified agent cgroup has memory quota set")
+        log.info("Successfully verified agent cgroup has no memory quota set")
     else:
-        fail("The agent's cgroup doesn't seem to have memory quota set. Agent Memory Quota: {0}".format(get_agent_memory_quota()))
+        fail("The agent's cgroup should not have a memory quota set, but MemoryHigh={0}".format(get_agent_memory_quota()))
 
 
 def verify_agent_reported_memory_metrics():
@@ -105,41 +105,43 @@ def check_agent_log_for_metrics() -> bool:
 
 def verify_memory_throttling_check_on_agent_cgroups():
     """
-    This method verifies that the agent detects memory throttling on its cgroup
+    This method verifies that the agent reports memory throttling metrics on its cgroup,
+    and that all reported values are zero. Since the agent no longer enforces a memory limit
+    (MemoryHigh=infinity), there should be no memory throttling events.
     """
-    log.info("** Verifying agent detected memory throttling on its cgroup")
+    log.info("** Verifying agent reports zero memory throttling on its cgroup")
 
+    # Only consider records logged from now on; older entries (e.g. from before this test
+    # configured the agent) must not influence the assertion.
     throttled_events = []
-    pressure_time = []
 
     def check_agent_log_for_metrics() -> bool:
+        # Rebuild on every attempt so the assertion reflects only the latest scan window
+        # and does not accumulate values across retries.
+        del throttled_events[:]
         for record in AgentLog().read():
             match = re.search(r"Memory/Total Memory Throttled Events \s*\[walinuxagent.service\]\s*=\s*([0-9.]+)", record.message)
             if match is not None:
                 throttled_events.append(float(match.group(1)))
-            else:
-                match = re.search(r"Memory/Memory Pressure \(s\)\s*\[walinuxagent.service\]\s*=\s*([0-9.]+)", record.message)
-                if match is not None:
-                    pressure_time.append(float(match.group(1)))
-        if len(pressure_time) < 1 or len(throttled_events) < 1:
-            return False
-        return True
+        return len(throttled_events) >= 1
 
-    distro = shellutil.run_command("get_distro.py").rstrip().lower()
+    found: bool = retry_if_false(check_agent_log_for_metrics, delay=60)
+    if not found:
+        fail(
+            "The agent doesn't seem to be collecting Memory Throttling metrics. Agent found Memory Throttle Events: {0}".format(
+                throttled_events))
 
-    if "rhel" in distro:
-        log.info("Skipping memory throttling check verification on RHEL distros due to known issues with memory pressure file not present.")
-        return
+    log.info("Memory Throttle Events: %s", throttled_events)
 
-    found: bool = retry_if_false(check_agent_log_for_metrics, delay=60)
-    if found:
-        log.info("Memory Throttle Events: %s", throttled_events)
-        log.info("Memory Pressure Time: %s", pressure_time)
-        log.info("Successfully verified agent reported memory throttling metrics")
-    else:
+    # No memory quota is enforced, so every reported value must be zero.
+    non_zero_throttled = [v for v in throttled_events if v != 0.0]
+    if non_zero_throttled:
         fail(
-            "The agent doesn't seem to be collecting Memory Throttling metrics. Agent found Memory Throttle Events: {0} and Pressure: {1}".format(
-                throttled_events, pressure_time))
+            "Expected all memory throttling values to be zero (no memory limit is enforced), "
+            "but found non-zero Memory Throttle Events: {0}".format(
+                non_zero_throttled))
+
+    log.info("Successfully verified agent reported zero memory throttling metrics")
 
 
 def cleanup_test_setup():
@@ -161,7 +163,7 @@ def cleanup_test_setup():
 def main():
     skip_if_distro_not_supports_memory_quota()
     prepare_agent()
-    verify_agent_has_memory_quota_set()
+    verify_agent_has_no_memory_quota_set()
     verify_agent_reported_memory_metrics()
     verify_memory_throttling_check_on_agent_cgroups()
     cleanup_test_setup()