Add post-test alerts

conftest.py

@@ -93,13 +93,13 @@
 
 TEAM_MARKERS = {
     "chaos": ["chaos", "deprecated_api"],
-    "virt": ["virt", "deprecated_api"],
-    "network": ["network", "deprecated_api"],
-    "storage": ["storage", "deprecated_api"],
-    "iuo": ["install_upgrade_operators", "deprecated_api"],
-    "observability": ["observability", "deprecated_api"],
-    "infrastructure": ["infrastructure", "deprecated_api"],
-    "data_protection": ["data_protection", "deprecated_api"],
+    "virt": ["virt", "deprecated_api", "post_test_alerts"],
+    "network": ["network", "deprecated_api", "post_test_alerts"],
+    "storage": ["storage", "deprecated_api", "post_test_alerts"],
+    "iuo": ["install_upgrade_operators", "deprecated_api", "post_test_alerts"],
+    "observability": ["observability", "deprecated_api", "post_test_alerts"],
+    "infrastructure": ["infrastructure", "deprecated_api", "post_test_alerts"],
+    "data_protection": ["data_protection", "deprecated_api", "post_test_alerts"],
 }
 NAMESPACE_COLLECTION = {
     "storage": [NamespacesNames.OPENSHIFT_STORAGE],
@@ -129,6 +129,7 @@ def pytest_addoption(parser):
     ci_group = parser.getgroup(name="CI")
     component_sanity_group = parser.getgroup(name="ComponentSanity")
     ai_insights_group = parser.getgroup(name="ai-job-insight")
+    post_test_alerts_group = parser.getgroup(name="PostTestAlerts")
 
     # Upgrade addoption
     install_upgrade_group.addoption(
@@ -298,6 +299,13 @@ def pytest_addoption(parser):
         action="store_true",
     )
 
+    # Post test alerts group
+    post_test_alerts_group.addoption(
+        "--skip-post-test-alerts",
+        help="By default test_no_critical_alerts_after_tests will always run, pass this flag to skip it",
+        action="store_true",
+    )
+
     # LeftoversCollector group
     leftovers_collector.addoption(
         "--leftovers-collector",
@@ -550,6 +558,20 @@ def filter_deprecated_api_tests(items: list[Item], config: Config) -> list[Item]
     return items
 
 
+def filter_post_test_alerts_tests(items: list[Item], config: Config) -> list[Item]:
+    # filter out post test alerts tests, if explicitly asked or if running upgrade/install tests
+    if (
+        config.getoption("--skip-post-test-alerts")
+        or config.getoption("--install")
+        or config.getoption("--upgrade")
+        or config.getoption("--upgrade_custom")
+    ):
+        discard_tests, items_to_return = remove_tests_from_list(items=items, filter_str="post_test_alerts")
+        config.hook.pytest_deselected(items=discard_tests)
+        return items_to_return
+    return items
+
+
 def filter_sno_only_tests(items: list[Item], config: Config) -> list[Item]:
     if config.getoption("-m") and "sno" not in config.getoption("-m"):
         discard_tests, items_to_return = remove_tests_from_list(items=items, filter_str="single_node_tests")
@@ -570,12 +592,19 @@ def remove_tests_from_list(items: list[Item], filter_str: str) -> tuple[list[Ite
 
 
 def pytest_configure(config):
+    config._test_execution_start_time = datetime.datetime.now(tz=datetime.UTC)
+
     # test_deprecation_audit_logs should always run regardless the path that passed to pytest.
     deprecation_tests_dir_path = "tests/deprecated_api"
     file_or_dir = config.option.file_or_dir
     if file_or_dir and deprecation_tests_dir_path not in file_or_dir and file_or_dir != ["tests"]:
         config.option.file_or_dir.append(deprecation_tests_dir_path)
 
+    # post_test_alerts tests should always run regardless the path that passed to pytest.
+    post_test_alerts_dir_path = "tests/post_test_alerts"
+    if file_or_dir and post_test_alerts_dir_path not in file_or_dir and file_or_dir != ["tests"]:
+        config.option.file_or_dir.append(post_test_alerts_dir_path)
+
     if conformance_storage_class := config.getoption("conformance_storage_class"):
         py_config["storage_class_matrix"] = StorageClassConfig(
             name=conformance_storage_class
@@ -634,6 +663,7 @@ def pytest_collection_modifyitems(session, config, items):
     if discard:
         config.hook.pytest_deselected(items=discard)
     items[:] = filter_deprecated_api_tests(items=items, config=config)
+    items[:] = filter_post_test_alerts_tests(items=items, config=config)
     items[:] = filter_sno_only_tests(items=items, config=config)
     items[:] = mark_nmstate_dependent_tests(items=items)
 

tests/post_test_alerts/test_post_test_alerts.py

@@ -0,0 +1,55 @@
+"""
+Post-test alerts verification.
+
+Verifies that critical CNV alerts were not triggered during test execution.
+
+Jira: https://redhat.atlassian.net/browse/CNV-80353
+"""
+
+import datetime
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+POST_TEST_CRITICAL_ALERTS = [
+    "LowVirtControllersCount",
+    "LowVirtAPICount",
+    "KubeVirtCRModified",
+    "VirtControllerRESTErrorsHigh",
+    "VirtHandlerRESTErrorsHigh",
+    "HCOOperatorConditionsUnhealthy",
+]
+
+ALERTS_REGEX = "|".join(POST_TEST_CRITICAL_ALERTS)
+
+
+@pytest.mark.s390x
+@pytest.mark.polarion("CNV-16276")
+@pytest.mark.order("last")
+def test_no_critical_alerts_after_tests(prometheus, request):
+    """
+    Test that critical CNV alerts were not triggered during test execution.
+
+    Preconditions:
+        - Prometheus is accessible on the cluster
+        - Test execution completed
+
+    Steps:
+        1. Query Prometheus for alerts that fired at any point during test execution
+        2. Check that none of the critical alerts were triggered
+
+    Expected:
+        - None of the critical alerts fired during test execution
+    """
+    start_time = request.config._test_execution_start_time
+    duration_seconds = int((datetime.datetime.now(tz=datetime.UTC) - start_time).total_seconds())
+    LOGGER.info(
+        f"Checking {len(POST_TEST_CRITICAL_ALERTS)} critical alerts"
+        f" were not triggered during test execution (last {duration_seconds}s)"
+    )
+    query = f'ALERTS{{alertname=~"{ALERTS_REGEX}", alertstate="firing"}}[{duration_seconds}s]'
+    results = prometheus.query_sampler(query=query)
+    fired_alerts = {result["metric"]["alertname"]: result for result in results}
+    assert not fired_alerts, f"Critical alerts fired during test execution.\n{fired_alerts}"