diff --git a/conftest.py b/conftest.py index 964b2c866c..4569d10c79 100644 --- a/conftest.py +++ b/conftest.py @@ -93,13 +93,13 @@ TEAM_MARKERS = { "chaos": ["chaos", "deprecated_api"], - "virt": ["virt", "deprecated_api"], - "network": ["network", "deprecated_api"], - "storage": ["storage", "deprecated_api"], - "iuo": ["install_upgrade_operators", "deprecated_api"], - "observability": ["observability", "deprecated_api"], - "infrastructure": ["infrastructure", "deprecated_api"], - "data_protection": ["data_protection", "deprecated_api"], + "virt": ["virt", "deprecated_api", "post_test_alerts"], + "network": ["network", "deprecated_api", "post_test_alerts"], + "storage": ["storage", "deprecated_api", "post_test_alerts"], + "iuo": ["install_upgrade_operators", "deprecated_api", "post_test_alerts"], + "observability": ["observability", "deprecated_api", "post_test_alerts"], + "infrastructure": ["infrastructure", "deprecated_api", "post_test_alerts"], + "data_protection": ["data_protection", "deprecated_api", "post_test_alerts"], } NAMESPACE_COLLECTION = { "storage": [NamespacesNames.OPENSHIFT_STORAGE], @@ -129,6 +129,7 @@ def pytest_addoption(parser): ci_group = parser.getgroup(name="CI") component_sanity_group = parser.getgroup(name="ComponentSanity") ai_insights_group = parser.getgroup(name="ai-job-insight") + post_test_alerts_group = parser.getgroup(name="PostTestAlerts") # Upgrade addoption install_upgrade_group.addoption( @@ -298,6 +299,13 @@ def pytest_addoption(parser): action="store_true", ) + # Post test alerts group + post_test_alerts_group.addoption( + "--skip-post-test-alerts", + help="By default test_no_critical_alerts_after_tests will always run, pass this flag to skip it", + action="store_true", + ) + # LeftoversCollector group leftovers_collector.addoption( "--leftovers-collector", @@ -550,6 +558,20 @@ def filter_deprecated_api_tests(items: list[Item], config: Config) -> list[Item] return items +def filter_post_test_alerts_tests(items: list[Item], config: Config) -> list[Item]: + # filter out post test alerts tests, if explicitly asked or if running upgrade/install tests + if ( + config.getoption("--skip-post-test-alerts") + or config.getoption("--install") + or config.getoption("--upgrade") + or config.getoption("--upgrade_custom") + ): + discard_tests, items_to_return = remove_tests_from_list(items=items, filter_str="post_test_alerts") + config.hook.pytest_deselected(items=discard_tests) + return items_to_return + return items + + def filter_sno_only_tests(items: list[Item], config: Config) -> list[Item]: if config.getoption("-m") and "sno" not in config.getoption("-m"): discard_tests, items_to_return = remove_tests_from_list(items=items, filter_str="single_node_tests") @@ -570,12 +592,19 @@ def remove_tests_from_list(items: list[Item], filter_str: str) -> tuple[list[Ite def pytest_configure(config): + config._test_execution_start_time = datetime.datetime.now(tz=datetime.UTC) + # test_deprecation_audit_logs should always run regardless the path that passed to pytest. deprecation_tests_dir_path = "tests/deprecated_api" file_or_dir = config.option.file_or_dir if file_or_dir and deprecation_tests_dir_path not in file_or_dir and file_or_dir != ["tests"]: config.option.file_or_dir.append(deprecation_tests_dir_path) + # post_test_alerts tests should always run regardless the path that passed to pytest. + post_test_alerts_dir_path = "tests/post_test_alerts" + if file_or_dir and post_test_alerts_dir_path not in file_or_dir and file_or_dir != ["tests"]: + config.option.file_or_dir.append(post_test_alerts_dir_path) + if conformance_storage_class := config.getoption("conformance_storage_class"): py_config["storage_class_matrix"] = StorageClassConfig( name=conformance_storage_class @@ -634,6 +663,7 @@ def pytest_collection_modifyitems(session, config, items): if discard: config.hook.pytest_deselected(items=discard) items[:] = filter_deprecated_api_tests(items=items, config=config) + items[:] = filter_post_test_alerts_tests(items=items, config=config) items[:] = filter_sno_only_tests(items=items, config=config) items[:] = mark_nmstate_dependent_tests(items=items) diff --git a/tests/post_test_alerts/__init__.py b/tests/post_test_alerts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/post_test_alerts/test_post_test_alerts.py b/tests/post_test_alerts/test_post_test_alerts.py new file mode 100644 index 0000000000..2f11962ab8 --- /dev/null +++ b/tests/post_test_alerts/test_post_test_alerts.py @@ -0,0 +1,55 @@ +""" +Post-test alerts verification. + +Verifies that critical CNV alerts were not triggered during test execution. + +Jira: https://redhat.atlassian.net/browse/CNV-80353 +""" + +import datetime +import logging + +import pytest + +LOGGER = logging.getLogger(__name__) + +POST_TEST_CRITICAL_ALERTS = [ + "LowVirtControllersCount", + "LowVirtAPICount", + "KubeVirtCRModified", + "VirtControllerRESTErrorsHigh", + "VirtHandlerRESTErrorsHigh", + "HCOOperatorConditionsUnhealthy", +] + +ALERTS_REGEX = "|".join(POST_TEST_CRITICAL_ALERTS) + + +@pytest.mark.s390x +@pytest.mark.polarion("CNV-16276") +@pytest.mark.order("last") +def test_no_critical_alerts_after_tests(prometheus, request): + """ + Test that critical CNV alerts were not triggered during test execution. + + Preconditions: + - Prometheus is accessible on the cluster + - Test execution completed + + Steps: + 1. Query Prometheus for alerts that fired at any point during test execution + 2. Check that none of the critical alerts were triggered + + Expected: + - None of the critical alerts fired during test execution + """ + start_time = request.config._test_execution_start_time + duration_seconds = int((datetime.datetime.now(tz=datetime.UTC) - start_time).total_seconds()) + LOGGER.info( + f"Checking {len(POST_TEST_CRITICAL_ALERTS)} critical alerts" + f" were not triggered during test execution (last {duration_seconds}s)" + ) + query = f'ALERTS{{alertname=~"{ALERTS_REGEX}", alertstate="firing"}}[{duration_seconds}s]' + results = prometheus.query_sampler(query=query) + fired_alerts = {result["metric"]["alertname"]: result for result in results} + assert not fired_alerts, f"Critical alerts fired during test execution.\n{fired_alerts}"