Fix static-checks loading and add error handling

Author: RichardPlesnik

operatorcert/operator_repo/checks/__init__.py

@@ -103,6 +103,7 @@ def get_checks(
     """
     if skip_tests is None:
         skip_tests = []
+    log.debug("Loading checks from suite: %s", suite_name)
     result: dict[str, list[Check]] = {}
     for module_name, _ in SUPPORTED_TYPES:
         result[module_name] = []
@@ -120,8 +121,36 @@ def get_checks(
                         log.debug("Skipping %s check", check_name)
                         continue
                     result[module_name].append(check)
-        except ModuleNotFoundError:
-            pass
+        except ModuleNotFoundError as e:
+            # ModuleNotFoundError can be caused by two different reasons:
+            # - file doesn't exist (e.g. community/operator_catalogs.py) - OK
+            # - module exists but has a missing import dependency - SHOULD FAIL
+
+            module_path = f"{suite_name}.{module_name}"
+            error_message = str(e)
+            if f"No module named '{module_path}'" in error_message:
+                # Nonexistent module
+                log.debug("Module %s not found: %s", module_path, e)
+            else:
+                # Issue with missing dependency
+                error_msg = f"Failed to import {module_path} due to error: {e}"
+                log.debug(error_msg)
+                traceback.print_exc()
+                raise RuntimeError(error_msg) from e
+        except Exception as e:
+            # Unexpected error - SHOULD FAIL
+            error_msg = (
+                f"Unexpected error loading module {suite_name}.{module_name}: {e}"
+            )
+            log.debug(error_msg)
+            traceback.print_exc()
+            raise RuntimeError(error_msg) from e
+
+    num_loaded = len([checks for checks in result.values() if checks])
+    log.debug("Loaded %d check types from %s", num_loaded, suite_name)
+    for module_type, checks in result.items():
+        if checks:
+            log.debug("  %s: %d checks", module_type, len(checks))
     return result
 
 

poetry.lock

@@ -34,6 +34,7 @@ dependencies = [
     "httpx (>=0.28.1)",
     "kubernetes (>=33.1.0)",
     "cel-python (>=0.4.0)",
+    "jsonschema (>=4.0.0)",
 ]
 
 [project.scripts]

pyproject.toml

@@ -66,7 +66,10 @@ def check_fake(_something):  # type: ignore
 
 @patch("importlib.import_module")
 def test_get_checks_missing_modules(mock_import_module: MagicMock) -> None:
-    mock_import_module.side_effect = ModuleNotFoundError()
+    def side_effect(module_name: str) -> None:
+        raise ModuleNotFoundError(f"No module named '{module_name}'")
+
+    mock_import_module.side_effect = side_effect
     assert get_checks("suite.name") == {
         "operator": [],
         "bundle": [],
@@ -77,6 +80,40 @@ def test_get_checks_missing_modules(mock_import_module: MagicMock) -> None:
     )
 
 
+@patch("importlib.import_module")
+def test_get_checks_missing_dependency(mock_import_module: MagicMock) -> None:
+    """Test that missing dependencies cause a RuntimeError"""
+
+    def side_effect(module_name: str) -> None:
+        # Simulate a missing dependency (e.g., jsonschema)
+        raise ModuleNotFoundError("No module named 'jsonschema'")
+
+    mock_import_module.side_effect = side_effect
+    try:
+        get_checks("suite.name")
+        assert False, "Expected RuntimeError to be raised"
+    except RuntimeError as e:
+        assert "Failed to import suite.name.operator due to error" in str(e)
+        assert "jsonschema" in str(e)
+
+
+@patch("importlib.import_module")
+def test_get_checks_unexpected_error(mock_import_module: MagicMock) -> None:
+    """Test that unexpected errors during module loading cause a RuntimeError"""
+
+    def side_effect(module_name: str) -> None:
+        # Simulate an unexpected error (e.g., SyntaxError, AttributeError)
+        raise ValueError("Unexpected error during import")
+
+    mock_import_module.side_effect = side_effect
+    try:
+        get_checks("suite.name")
+        assert False, "Expected RuntimeError to be raised"
+    except RuntimeError as e:
+        assert "Unexpected error loading module suite.name.operator" in str(e)
+        assert "Unexpected error during import" in str(e)
+
+
 def test_run_check(mock_bundle: Bundle) -> None:
     def check_fake(_something):  # type: ignore
         yield Warn("foo")