Expose user enabled features as pytest markers

This allows guarding tests behind markers.

Author: ekohl

docs/developer/testing.md

@@ -146,6 +146,32 @@ def test_service_port(server):
     assert server.addr("localhost").port(6379).is_reachable
 ```
 
+### Feature guarding
+
+Some functionality can only be tested when a feature is enabled.
+
+You can mark an individual test to be skipped if needed:
+
+```python
+@pytest.mark.feature("iop")
+def test_ingress_service(server):
+    service = server.service("iop-core-ingress")
+    assert service.is_running and service.is_enabled
+```
+
+Often it's better to have an entire file dedicated to a feature and mark the entire file as guarded.
+
+```python
+pytestmark = pytest.mark.feature("iop")
+
+def test_ingress_service(server):
+    service = server.service("iop-core-ingress")
+    assert service.is_running and service.is_enabled
+
+def test_ingress_http_endpoint(server):
+    # ...
+```
+
 ### API test
 
 The `foremanapi` fixture is an [apypie](https://github.com/Apipie/apypie) `ForemanApi` client that connects to the deployed Foreman instance(authenticated as `admin`/`changeme`). It maps directly to the Foreman REST API — each method takes a resource name that corresponds to an API endpoint:

tests/conftest.py

@@ -1,5 +1,6 @@
-import os
 import uuid
+import subprocess
+from functools import cached_property
 
 import apypie
 import paramiko
@@ -16,11 +17,40 @@
 SSH_CONFIG = './.tmp/ssh-config'
 
 
+class UserParameters:
+    def __init__(self, config):
+        self._config = config
+
+    @cached_property
+    def features(self):
+        # foremanctl outputs
+        # FEATURE                   STATE               DESCRIPTION
+        # $feature                  enabled/available   $description
+        output = subprocess.check_output(['./foremanctl', 'features'], cwd=self._config.rootdir,
+                                         universal_newlines=True)
+        lines = output.splitlines(keepends=False)
+        # feature, status, description
+        return [line.split(None, 2) for line in lines[1:]]
+
+    @cached_property
+    def available_features(self):
+        return set(feature for feature, _status, _desc in self.features)
+
+    @cached_property
+    def enabled_features(self):
+        return set(feature for feature, status, _desc in self.features if status == 'enabled')
+
+
 def pytest_addoption(parser):
     parser.addoption("--certificate-source", action="store", default="default", choices=('default', 'installer', 'custom_server'), help="Certificate source used during deployment")
     parser.addoption("--database-mode", action="store", default="internal", choices=('internal', 'external'), help="Whether the database is internal or external")
 
 
+@pytest.fixture(scope="module")
+def enabled_features(pytestconfig):
+    return pytestconfig.user_parameters.enabled_features
+
+
 @pytest.fixture(scope="module")
 def fixture_dir():
     return py.path.local(__file__).realpath() / '..' / 'fixtures'
@@ -206,35 +236,21 @@ def wait_for_metadata_generate(foremanapi):
     wait_for_tasks(foremanapi, 'label = Actions::Katello::Repository::MetadataGenerate')
 
 
-def enabled_features():
-    test_dir = os.path.dirname(os.path.abspath(__file__))
-    foremanctl_dir = os.path.dirname(test_dir)
-    params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml')
-    if os.path.exists(params_file):
-        with open(params_file, 'r') as f:
-            features = yaml.safe_load(f).get('features', [])
-            if isinstance(features, str):
-                features = features.split()
-            return features
-    return []
-
-
-def is_iop_enabled():
-    return 'iop' in enabled_features()
-
-
 def pytest_configure(config):
-    config.addinivalue_line("markers", "iop: tests requiring IOP to be enabled")
+    config.addinivalue_line("markers", "feature(name): mark a test as requiring a feature")
 
+    config.user_parameters = UserParameters(config)
 
-def pytest_collection_modifyitems(config, items):
-    if is_iop_enabled():
-        return
 
-    skip_iop = pytest.mark.skip(reason="IOP not enabled - skipping IOP tests ('iop' not in enabled_features)")
-    for item in items:
-        if "iop" in item.keywords:
-            item.add_marker(skip_iop)
+def pytest_runtest_setup(item):
+    feature_markers = set(mark.args[0] for mark in item.iter_markers(name="feature"))
+    if feature_markers:
+        invalid_features = feature_markers - item.config.user_parameters.available_features
+        if invalid_features:
+            raise pytest.PytestConfigWarning(f"Invalid feature(s) {invalid_features!r} on {item}")
+        missing = feature_markers - item.config.user_parameters.enabled_features
+        if missing:
+            pytest.skip(f"test requires feature(s) {missing!r}")
 
 
 class ResolveAdapter(HTTPAdapter):

tests/foreman_proxy_test.py

@@ -2,15 +2,10 @@
 import json
 
 import pytest
-from conftest import enabled_features
 
 FOREMAN_PROXY_PORT = 8443
 
 
-def is_bmc_enabled():
-    return 'bmc' in enabled_features()
-
-
 def get_proxy_v2_features(server, certificates, server_fqdn):
     cmd = server.run(
         f"curl --cacert {certificates['server_ca_certificate']} "
@@ -22,14 +17,14 @@ def get_proxy_v2_features(server, certificates, server_fqdn):
     return json.loads(cmd.stdout)
 
 
-def test_foreman_proxy_features(server, certificates, server_fqdn):
+def test_foreman_proxy_features(server, certificates, server_fqdn, enabled_features):
     cmd = server.run(f"curl --cacert {certificates['server_ca_certificate']} --silent https://{server_fqdn}:{FOREMAN_PROXY_PORT}/features")
     assert cmd.succeeded
     features = json.loads(cmd.stdout)
     assert "logs" in features
     assert "script" in features
     assert "dynflow" in features
-    if is_bmc_enabled():
+    if 'bmc' in enabled_features:
         assert "bmc" in features
     else:
         assert "bmc" not in features
@@ -60,7 +55,7 @@ def test_foreman_proxy_client_auth_to_foreman(server, certificates, server_fqdn)
     assert cmd.stdout == '201'
 
 
-@pytest.mark.skipif("not is_bmc_enabled()")
+@pytest.mark.feature('bmc')
 def test_bmc_capabilities(server, certificates, server_fqdn):
     features = get_proxy_v2_features(server, certificates, server_fqdn)
     assert 'bmc' in features

tests/iop/test_advisor.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_advisor_backend_api_service(server):

tests/iop/test_advisor_frontend.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_advisor_frontend_assets_directory(server):

tests/iop/test_cvemap_downloader.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_cvemap_download_script(server):

tests/iop/test_engine.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_engine_service(server):

tests/iop/test_gateway.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_gateway_service(server):

tests/iop/test_ingress.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_ingress_service(server):

tests/iop/test_integration.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_iop_core_kafka_service(server):