Expose user enabled features as pytest markers

This allows guarding tests behind markers.

Author: ekohl

docs/developer/testing.md

@@ -146,6 +146,32 @@ def test_service_port(server):
     assert server.addr("localhost").port(6379).is_reachable
 ```
 
+### Feature guarding
+
+Some functionality can only be tested when a feature is enabled.
+
+You can mark an invidivual test to be skipped if needed:
+
+```python
+@pytest.mark.feature("iop")
+def test_ingress_service(server):
+    service = server.service("iop-core-ingress")
+    assert service.is_running and service.is_enabled
+```
+
+Often it's better to have an entire file dedicated to a feature and mark the entire file as guarded.
+
+```python
+pytestmark = pytest.mark.feature("iop")
+
+def test_ingress_service(server):
+    service = server.service("iop-core-ingress")
+    assert service.is_running and service.is_enabled
+
+def test_ingress_http_endpoint(server):
+    # ...
+```
+
 ### API test
 
 The `foremanapi` fixture is an [apypie](https://github.com/Apipie/apypie) `ForemanApi` client that connects to the deployed Foreman instance(authenticated as `admin`/`changeme`). It maps directly to the Foreman REST API — each method takes a resource name that corresponds to an API endpoint:

tests/conftest.py

@@ -1,5 +1,7 @@
 import os
 import uuid
+from functools import cached_property
+from pathlib import Path
 
 import apypie
 import paramiko
@@ -16,6 +18,38 @@
 SSH_CONFIG='./.tmp/ssh-config'
 
 
+class UserParameters:
+    def __init__(self, config):
+        self._config = config
+
+    @property
+    def _obsah_state(self) -> Path:
+        # mirror what foremanctl / obsah does
+        if state := os.environ.get('OBSAH_STATE'):
+            return Path(state)
+
+        if base := os.environ.get('OBSAH_BASE'):
+            root = Path(base)
+        else:
+            root = self._config.rootpath
+        return root / '.var' / 'lib' / 'foremanctl'
+
+    def _read_parameters(self):
+        params_file = self._obsah_state / 'parameters.yaml'
+        if params_file.exists():
+            with params_file.open('r') as f:
+                return yaml.safe_load(f)
+
+        return None
+
+    @cached_property
+    def features(self):
+        params = self._read_parameters()
+        if params:
+            return set(params.get('features', []))
+        return set()
+
+
 def pytest_addoption(parser):
     parser.addoption("--certificate-source", action="store", default="default", choices=('default', 'installer', 'custom_server'), help="Certificate source used during deployment")
     parser.addoption("--database-mode", action="store", default="internal", choices=('internal', 'external'), help="Whether the database is internal or external")
@@ -206,34 +240,18 @@ def wait_for_metadata_generate(foremanapi):
     wait_for_tasks(foremanapi, 'label = Actions::Katello::Repository::MetadataGenerate')
 
 
-def is_iop_enabled():
-    test_dir = os.path.dirname(os.path.abspath(__file__))
-    foremanctl_dir = os.path.dirname(test_dir)
-    params_file = os.path.join(foremanctl_dir, '.var', 'lib', 'foremanctl', 'parameters.yaml')
-
-    if os.path.exists(params_file):
-        with open(params_file, 'r') as f:
-            params = yaml.safe_load(f)
-            features = params.get('features', [])
-            if isinstance(features, str):
-                features = features.split()
-            return 'iop' in features
-
-    return False
-
-
 def pytest_configure(config):
-    config.addinivalue_line("markers", "iop: tests requiring IOP to be enabled")
+    config.addinivalue_line("markers", "feature(name): mark a test as requiring a feature")
 
+    config.user_parameters = UserParameters(config)
 
-def pytest_collection_modifyitems(config, items):
-    if is_iop_enabled():
-        return
 
-    skip_iop = pytest.mark.skip(reason="IOP not enabled - skipping IOP tests ('iop' not in enabled_features)")
-    for item in items:
-        if "iop" in item.keywords:
-            item.add_marker(skip_iop)
+def pytest_runtest_setup(item):
+    feature_markers = set(mark.args[0] for mark in item.iter_markers(name="feature"))
+    if feature_markers:
+        missing = feature_markers - item.config.user_parameters.features
+        if missing:
+            pytest.skip("test requires feature(s) {!r}".format(missing))
 
 
 class ResolveAdapter(HTTPAdapter):

tests/iop/test_advisor.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_advisor_backend_api_service(server):

tests/iop/test_advisor_frontend.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_advisor_frontend_assets_directory(server):

tests/iop/test_cvemap_downloader.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_cvemap_download_script(server):

tests/iop/test_engine.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_engine_service(server):

tests/iop/test_gateway.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_gateway_service(server):

tests/iop/test_ingress.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_ingress_service(server):

tests/iop/test_integration.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_iop_core_kafka_service(server):

tests/iop/test_inventory.py

@@ -1,6 +1,6 @@
 import pytest
 
-pytestmark = pytest.mark.iop
+pytestmark = pytest.mark.feature("iop")
 
 
 def test_inventory_migrate_service(server):