chore(test): poll for alert store expiry instead of fixed sleeps

test_monitorclient_remove_old_alerts read the alert store once at a
fixed offset after starting mender-monitor and asserted an exact key
count. mender-monitor purges expired records on a periodic loop with
variable restart latency, so the single timed read raced the purge
cycle. The faster mender_device.run() from the SSH backoff change
(no longer sleeping 1s before every healthy command) shifted the read
~2s earlier and collapsed the timing margin, surfacing the latent
flakiness (seen as '4' == '2' and '1' == '0').

Poll until the store drains to the expected count instead. It drains
monotonically (4 -> 2 -> 0), so 1s polling reliably observes each
plateau regardless of purge-loop latency.

Signed-off-by: pasinskim <marcin.pasinski@northern.tech>

Author: pasinskim

tests/tests/test_monitor_client.py

@@ -1593,30 +1593,41 @@ def test_monitorclient_remove_old_alerts(self, monitor_commercial_setup_no_clien
         )
         mender_device.run("systemctl start mender-monitor")
 
-        # T8: mender-monitor started
-        time.sleep(alert_resend_interval_s)
-        time.sleep(alert_resend_interval_s)
-
-        # Shift by 1s to avoid race condition when checking
-        time.sleep(1)
+        def count_alert_keys():
+            output = mender_device.run(
+                "bash -c 'cd /usr/share/mender-monitor && . lib/fixlenstore-lib.sh;"
+                + "keys_nolock | wc -l;'"
+            )
+            return int(output.strip())
+
+        def wait_for_alert_count(expected, max_wait_s):
+            # mender-monitor purges expired records on a periodic loop
+            # (DEFAULT_ALERT_STORE_RESEND_INTERVAL_S) whose restart latency
+            # varies, so poll for the expected count instead of reading once at a
+            # fixed offset, which races the purge cycle. The store drains
+            # monotonically (4 -> 2 -> 0), so 1s polling reliably observes each
+            # plateau. (QA-1527)
+            deadline = time.monotonic() + max_wait_s
+            count = None
+            while time.monotonic() < deadline:
+                count = count_alert_keys()
+                logger.info(
+                    "test_monitorclient_remove_old_alerts: %d keys in store" % count
+                )
+                if count == expected:
+                    return
+                time.sleep(1)
+            assert count == expected, "expected %d alert keys in store, got %s" % (
+                expected,
+                count,
+            )
 
-        # T16+1: key1, key2 expired
-        output = mender_device.run(
-            "bash -c 'cd /usr/share/mender-monitor && . lib/fixlenstore-lib.sh;"
-            + "keys_nolock | wc -l;'"
-        )
-        logger.info("test_monitorclient_remove_old_alerts got %s keys" % output)
-        assert output == "2\n"
+        # key1, key2 expire at alert_max_age, ~8s before key3, key4 (inserted 8s
+        # later), leaving a wide window where exactly 2 keys remain.
+        wait_for_alert_count(2, max_wait_s=2 * alert_max_age)
 
-        time.sleep(alert_resend_interval_s)
-        time.sleep(alert_resend_interval_s)
-        # T24+1: key3, key4 expired
-        output = mender_device.run(
-            "bash -c 'cd /usr/share/mender-monitor && . lib/fixlenstore-lib.sh;"
-            + "keys_nolock | wc -l;'"
-        )
-        logger.info("test_monitorclient_remove_old_alerts got %s keys" % output)
-        assert output == "0\n"
+        # key3, key4 expire ~8s after key1, key2, draining the store completely.
+        wait_for_alert_count(0, max_wait_s=2 * alert_max_age)
 
         mender_device.run(
             "mv /usr/share/mender-monitor/config/config.sh.backup /usr/share/mender-monitor/config/config.sh"