From 484db23680029f664ba07dab4395c062ef00ac95 Mon Sep 17 00:00:00 2001 From: albarker-rh Date: Mon, 15 Jun 2026 11:21:08 -0400 Subject: [PATCH 1/4] feat(post-test-alerts): add post-test critical alerts verification Verifies 7 critical CNV alerts were not triggered during test execution. Follows the deprecated_api auto-inclusion pattern. Refs: CNV-80353 Signed-off-by: albarker-rh Co-Authored-By: Claude Opus 4.6 --- conftest.py | 44 +++++++++++++--- tests/post_test_alerts/__init__.py | 0 .../post_test_alerts/test_post_test_alerts.py | 50 +++++++++++++++++++ 3 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 tests/post_test_alerts/__init__.py create mode 100644 tests/post_test_alerts/test_post_test_alerts.py diff --git a/conftest.py b/conftest.py index 964b2c866c..e9652b6d28 100644 --- a/conftest.py +++ b/conftest.py @@ -92,14 +92,14 @@ ] TEAM_MARKERS = { - "chaos": ["chaos", "deprecated_api"], - "virt": ["virt", "deprecated_api"], - "network": ["network", "deprecated_api"], - "storage": ["storage", "deprecated_api"], - "iuo": ["install_upgrade_operators", "deprecated_api"], - "observability": ["observability", "deprecated_api"], - "infrastructure": ["infrastructure", "deprecated_api"], - "data_protection": ["data_protection", "deprecated_api"], + "chaos": ["chaos", "deprecated_api", "post_test_alerts"], + "virt": ["virt", "deprecated_api", "post_test_alerts"], + "network": ["network", "deprecated_api", "post_test_alerts"], + "storage": ["storage", "deprecated_api", "post_test_alerts"], + "iuo": ["install_upgrade_operators", "deprecated_api", "post_test_alerts"], + "observability": ["observability", "deprecated_api", "post_test_alerts"], + "infrastructure": ["infrastructure", "deprecated_api", "post_test_alerts"], + "data_protection": ["data_protection", "deprecated_api", "post_test_alerts"], } NAMESPACE_COLLECTION = { "storage": [NamespacesNames.OPENSHIFT_STORAGE], @@ -129,6 +129,7 @@ def pytest_addoption(parser): ci_group = parser.getgroup(name="CI") component_sanity_group = parser.getgroup(name="ComponentSanity") ai_insights_group = parser.getgroup(name="ai-job-insight") + post_test_alerts_group = parser.getgroup(name="PostTestAlerts") # Upgrade addoption install_upgrade_group.addoption( @@ -298,6 +299,13 @@ def pytest_addoption(parser): action="store_true", ) + # Post test alerts group + post_test_alerts_group.addoption( + "--skip-post-test-alerts", + help="By default test_no_critical_alerts_after_tests will always run, pass this flag to skip it", + action="store_true", + ) + # LeftoversCollector group leftovers_collector.addoption( "--leftovers-collector", @@ -550,6 +558,20 @@ def filter_deprecated_api_tests(items: list[Item], config: Config) -> list[Item] return items +def filter_post_test_alerts_tests(items: list[Item], config: Config) -> list[Item]: + # filter out post test alerts tests, if explicitly asked or if running upgrade/install tests + if ( + config.getoption("--skip-post-test-alerts") + or config.getoption("--install") + or config.getoption("--upgrade") + or config.getoption("--upgrade_custom") + ): + discard_tests, items_to_return = remove_tests_from_list(items=items, filter_str="post_test_alerts") + config.hook.pytest_deselected(items=discard_tests) + return items_to_return + return items + + def filter_sno_only_tests(items: list[Item], config: Config) -> list[Item]: if config.getoption("-m") and "sno" not in config.getoption("-m"): discard_tests, items_to_return = remove_tests_from_list(items=items, filter_str="single_node_tests") @@ -576,6 +598,11 @@ def pytest_configure(config): if file_or_dir and deprecation_tests_dir_path not in file_or_dir and file_or_dir != ["tests"]: config.option.file_or_dir.append(deprecation_tests_dir_path) + # post_test_alerts tests should always run regardless the path that passed to pytest. + post_test_alerts_dir_path = "tests/post_test_alerts" + if file_or_dir and post_test_alerts_dir_path not in file_or_dir and file_or_dir != ["tests"]: + config.option.file_or_dir.append(post_test_alerts_dir_path) + if conformance_storage_class := config.getoption("conformance_storage_class"): py_config["storage_class_matrix"] = StorageClassConfig( name=conformance_storage_class @@ -634,6 +661,7 @@ def pytest_collection_modifyitems(session, config, items): if discard: config.hook.pytest_deselected(items=discard) items[:] = filter_deprecated_api_tests(items=items, config=config) + items[:] = filter_post_test_alerts_tests(items=items, config=config) items[:] = filter_sno_only_tests(items=items, config=config) items[:] = mark_nmstate_dependent_tests(items=items) diff --git a/tests/post_test_alerts/__init__.py b/tests/post_test_alerts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/post_test_alerts/test_post_test_alerts.py b/tests/post_test_alerts/test_post_test_alerts.py new file mode 100644 index 0000000000..af2569df9d --- /dev/null +++ b/tests/post_test_alerts/test_post_test_alerts.py @@ -0,0 +1,50 @@ +""" +Post-test alerts verification. + +Verifies that critical CNV alerts were not triggered during test execution. + +Jira: CNV-80353 +""" + +import logging + +import pytest + +LOGGER = logging.getLogger(__name__) + +POST_TEST_CRITICAL_ALERTS = [ + "KubeVirtDeprecatedAPIRequested", + "LowVirtControllersCount", + "LowVirtAPICount", + "KubeVirtCRModified", + "VirtControllerRESTErrorsHigh", + "VirtHandlerRESTErrorsHigh", + "HCOOperatorConditionsUnhealthy", +] + + +@pytest.mark.s390x +@pytest.mark.polarion("CNV-80353") +@pytest.mark.order("last") +def test_no_critical_alerts_after_tests(prometheus): + """ + Test that critical CNV alerts were not triggered during test execution. + + Preconditions: + - Prometheus is accessible on the cluster + - Test execution completed + + Steps: + 1. Query Prometheus for each alert in the critical alerts list + 2. Check that none of the listed alerts are in firing state + + Expected: + - None of the critical alerts are firing + """ + LOGGER.info(f"Checking {len(POST_TEST_CRITICAL_ALERTS)} critical alerts were not triggered during test execution") + fired_alerts = {} + for alert_name in POST_TEST_CRITICAL_ALERTS: + alerts_by_name = prometheus.get_all_alerts_by_alert_name(alert_name=alert_name) + if alerts_by_name and alerts_by_name[0]["state"] == "firing": + fired_alerts[alert_name] = alerts_by_name + assert not fired_alerts, f"Critical alerts should not be fired after test execution.\n{fired_alerts}" From fb472f536d2a5d2c9d0c95270c7f4fd8280a513d Mon Sep 17 00:00:00 2001 From: albarker-rh Date: Mon, 15 Jun 2026 13:07:02 -0400 Subject: [PATCH 2/4] added jira url, fixed polarion, fixed test potentially missing other fired alerts Signed-off-by: albarker-rh --- tests/post_test_alerts/test_post_test_alerts.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/post_test_alerts/test_post_test_alerts.py b/tests/post_test_alerts/test_post_test_alerts.py index af2569df9d..ffeb1fe301 100644 --- a/tests/post_test_alerts/test_post_test_alerts.py +++ b/tests/post_test_alerts/test_post_test_alerts.py @@ -3,7 +3,7 @@ Verifies that critical CNV alerts were not triggered during test execution. -Jira: CNV-80353 +Jira: https://redhat.atlassian.net/browse/CNV-80353 """ import logging @@ -24,7 +24,7 @@ @pytest.mark.s390x -@pytest.mark.polarion("CNV-80353") +@pytest.mark.polarion("CNV-16276") @pytest.mark.order("last") def test_no_critical_alerts_after_tests(prometheus): """ @@ -45,6 +45,7 @@ def test_no_critical_alerts_after_tests(prometheus): fired_alerts = {} for alert_name in POST_TEST_CRITICAL_ALERTS: alerts_by_name = prometheus.get_all_alerts_by_alert_name(alert_name=alert_name) - if alerts_by_name and alerts_by_name[0]["state"] == "firing": + firing_alerts = [alert for alert in alerts_by_name if alert.get("state") == "firing"] + if firing_alerts: fired_alerts[alert_name] = alerts_by_name assert not fired_alerts, f"Critical alerts should not be fired after test execution.\n{fired_alerts}" From 106cf66cb33d2d6db776b57ffbce129e3f948de0 Mon Sep 17 00:00:00 2001 From: albarker-rh Date: Tue, 16 Jun 2026 12:17:50 -0400 Subject: [PATCH 3/4] removed post_test_alerts from chaos team marker Signed-off-by: albarker-rh --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index e9652b6d28..7fa7952a76 100644 --- a/conftest.py +++ b/conftest.py @@ -92,7 +92,7 @@ ] TEAM_MARKERS = { - "chaos": ["chaos", "deprecated_api", "post_test_alerts"], + "chaos": ["chaos", "deprecated_api"], "virt": ["virt", "deprecated_api", "post_test_alerts"], "network": ["network", "deprecated_api", "post_test_alerts"], "storage": ["storage", "deprecated_api", "post_test_alerts"], From 4046061a719b9f627a9cedb62da635f69845ef0d Mon Sep 17 00:00:00 2001 From: albarker-rh Date: Thu, 18 Jun 2026 11:11:10 -0400 Subject: [PATCH 4/4] fixed test to check alerts across test execution period Signed-off-by: albarker-rh --- conftest.py | 2 ++ .../post_test_alerts/test_post_test_alerts.py | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/conftest.py b/conftest.py index 7fa7952a76..4569d10c79 100644 --- a/conftest.py +++ b/conftest.py @@ -592,6 +592,8 @@ def remove_tests_from_list(items: list[Item], filter_str: str) -> tuple[list[Ite def pytest_configure(config): + config._test_execution_start_time = datetime.datetime.now(tz=datetime.UTC) + # test_deprecation_audit_logs should always run regardless the path that passed to pytest. deprecation_tests_dir_path = "tests/deprecated_api" file_or_dir = config.option.file_or_dir diff --git a/tests/post_test_alerts/test_post_test_alerts.py b/tests/post_test_alerts/test_post_test_alerts.py index ffeb1fe301..2f11962ab8 100644 --- a/tests/post_test_alerts/test_post_test_alerts.py +++ b/tests/post_test_alerts/test_post_test_alerts.py @@ -6,6 +6,7 @@ Jira: https://redhat.atlassian.net/browse/CNV-80353 """ +import datetime import logging import pytest @@ -13,7 +14,6 @@ LOGGER = logging.getLogger(__name__) POST_TEST_CRITICAL_ALERTS = [ - "KubeVirtDeprecatedAPIRequested", "LowVirtControllersCount", "LowVirtAPICount", "KubeVirtCRModified", @@ -22,11 +22,13 @@ "HCOOperatorConditionsUnhealthy", ] +ALERTS_REGEX = "|".join(POST_TEST_CRITICAL_ALERTS) + @pytest.mark.s390x @pytest.mark.polarion("CNV-16276") @pytest.mark.order("last") -def test_no_critical_alerts_after_tests(prometheus): +def test_no_critical_alerts_after_tests(prometheus, request): """ Test that critical CNV alerts were not triggered during test execution. @@ -35,17 +37,19 @@ def test_no_critical_alerts_after_tests(prometheus): - Test execution completed Steps: - 1. Query Prometheus for each alert in the critical alerts list - 2. Check that none of the listed alerts are in firing state + 1. Query Prometheus for alerts that fired at any point during test execution + 2. Check that none of the critical alerts were triggered Expected: - - None of the critical alerts are firing + - None of the critical alerts fired during test execution """ - LOGGER.info(f"Checking {len(POST_TEST_CRITICAL_ALERTS)} critical alerts were not triggered during test execution") - fired_alerts = {} - for alert_name in POST_TEST_CRITICAL_ALERTS: - alerts_by_name = prometheus.get_all_alerts_by_alert_name(alert_name=alert_name) - firing_alerts = [alert for alert in alerts_by_name if alert.get("state") == "firing"] - if firing_alerts: - fired_alerts[alert_name] = alerts_by_name - assert not fired_alerts, f"Critical alerts should not be fired after test execution.\n{fired_alerts}" + start_time = request.config._test_execution_start_time + duration_seconds = int((datetime.datetime.now(tz=datetime.UTC) - start_time).total_seconds()) + LOGGER.info( + f"Checking {len(POST_TEST_CRITICAL_ALERTS)} critical alerts" + f" were not triggered during test execution (last {duration_seconds}s)" + ) + query = f'ALERTS{{alertname=~"{ALERTS_REGEX}", alertstate="firing"}}[{duration_seconds}s]' + results = prometheus.query_sampler(query=query) + fired_alerts = {result["metric"]["alertname"]: result for result in results} + assert not fired_alerts, f"Critical alerts fired during test execution.\n{fired_alerts}"