[ISV-5281] [ISV-5282] Integration tests (#838)

* [ISV-5281] [ISV-5282] Integration tests

- Implemented integration test main loop
- Partially implemented a basic test case

---------

Signed-off-by: Maurizio Porrato <mporrato@redhat.com>

Author: mporrato

.coveragerc

@@ -1,4 +1,5 @@
 [run]
 source = operator-pipeline-images/operatorcert
-omit = operator-pipeline-images/operatorcert/webhook/*
-
+omit =
+    operator-pipeline-images/operatorcert/webhook/*
+    operator-pipeline-images/operatorcert/integration/testcases/*

integration-tests-config-sample.yaml

@@ -1,5 +1,9 @@
 ---
 
+fixtures_repository:
+    # The GitHub repo containing fixtures used by test cases
+    url: https://github.com/foo/operators-integration-tests-fixtures
+    token: secretABC
 operator_repository:
     # The GitHub repository hosting the operators for integration tests
     url: https://github.com/foo/operators-integration-tests

operator-pipeline-images/operatorcert/entrypoints/integration_tests.py

@@ -9,6 +9,7 @@
 
 from operatorcert.integration.runner import run_integration_tests
 
+
 LOGGER = logging.getLogger("operator-cert")
 
 

operator-pipeline-images/operatorcert/integration/config.py

@@ -46,10 +46,18 @@ class Config(BaseModel):
     Root configuration object
     """
 
+    # GitHub repo containing fixtures used by test cases
+    fixtures_repository: GitHubRepoConfig
+    # Main GitHub repo to be used by test cases. PRs submitted to this repo will trigger
+    # the hosted pipeline
     operator_repository: GitHubRepoConfig
+    # GitHub repo to submit PRs from
     contributor_repository: GitHubRepoConfig
+    # Container registry where to store bundle and index images created by test cases
     bundle_registry: ContainerRegistryConfig
+    # Container registry where to push the operator-pipeline image
     test_registry: ContainerRegistryConfig
+    # The IIB instance to be used by integration tests
     iib: IIBConfig
 
     @classmethod

operator-pipeline-images/operatorcert/integration/runner.py

@@ -12,6 +12,7 @@
 
 from operatorcert.integration.config import Config
 from operatorcert.integration.external_tools import Ansible, Podman, Secret
+from operatorcert.integration.testcase import run_tests
 
 LOGGER = logging.getLogger("operator-cert")
 
@@ -98,14 +99,14 @@ def run_integration_tests(
         return 1
     LOGGER.debug("Connected to %s", oc_context.strip())
     cfg = Config.from_yaml(config_path)
-    test_id = datetime.now().strftime("%Y%m%d%H%M%S")
-    namespace = f"inttest-{test_id}"
+    run_id = datetime.now().strftime("%Y%m%d%H%M%S")
+    namespace = f"inttest-{run_id}"
     try:
         if image:
             tagged_image_name = image
         else:
             image_name = f"{cfg.test_registry.base_ref}/operator-pipeline-images"
-            tagged_image_name = f"{image_name}:{test_id}"
+            tagged_image_name = f"{image_name}:{run_id}"
             auth = (
                 {image_name: (cfg.test_registry.username, cfg.test_registry.password)}
                 if cfg.test_registry.username and cfg.test_registry.password
@@ -117,6 +118,7 @@ def run_integration_tests(
                 auth=auth,
             )
         _deploy_pipelines(project_dir, namespace, tagged_image_name)
+        run_tests(cfg, run_id)
     except subprocess.CalledProcessError as e:
         LOGGER.error("Command %s failed:", e.cmd)
         for line in e.stdout.decode("utf-8").splitlines():

operator-pipeline-images/operatorcert/integration/testcase.py

@@ -5,9 +5,10 @@
 import logging
 from typing import TypeVar
 
-from operatorcert.integration.config import Config
 from colorama import Fore, Style, init as colorama_init
 
+from operatorcert.integration.config import Config
+from operatorcert.integration.testcases import import_testcases
 
 LOGGER = logging.getLogger("operator-cert")
 
@@ -20,6 +21,7 @@ class BaseTestCase:
     def __init__(self, config: Config, logger: logging.Logger) -> None:
         self.config = config
         self.logger = logger
+        self.run_id = "(unset)"
 
     def setup(self) -> None:
         """
@@ -48,11 +50,12 @@ def cleanup(self) -> None:
         any resources created during the execution of the test
         """
 
-    def run(self) -> None:
+    def run(self, run_id: str) -> None:
         """
         Execute the test case; `setup()`, `watch()`, `validate()` and
         `cleanup()` are called in order
         """
+        self.run_id = run_id
         try:
             self.setup()
             self.watch()
@@ -77,22 +80,23 @@ def integration_test_case(test_class: _T) -> _T:
     return test_class
 
 
-def run_tests(config: Config) -> int:
+def run_tests(config: Config, run_id: str) -> int:
     """
     Executes all the test cases that have been registered using the
     `integration_test_case` decorator
 
     Return:
         number of test cases that failed
     """
+    import_testcases()
     colorama_init()
     failed = 0
     for test_class in _test_cases:
         test_name = test_class.__name__
         print(f"Running {test_name} ", end="")
         try:
             test_instance = test_class(config, LOGGER)
-            test_instance.run()
+            test_instance.run(run_id)
             print(f"{Fore.GREEN}PASS{Style.RESET_ALL}")
         except Exception as e:  # pylint: disable=broad-except
             print(f"{Fore.RED}FAIL{Style.RESET_ALL}")

operator-pipeline-images/operatorcert/integration/testcases/__init__.py

@@ -0,0 +1,50 @@
+"""
+Import all submodules, so that all testcases are available.
+"""
+
+import importlib
+import pkgutil
+from types import ModuleType
+
+
+def import_submodules(
+    package: str | ModuleType, recursive: bool = True
+) -> dict[str, ModuleType]:
+    """Import all submodules of a module, recursively, including subpackages.
+
+    This function dynamically imports all submodules found within a given package,
+    optionally including subpackages if recursive mode is enabled. It's useful for
+    automatically loading all modules in a package without explicitly importing each one.
+
+    Args:
+        package (str | ModuleType): The package to import submodules from. Can be either:
+            - A string representing the module name (e.g., 'mypackage.subpackage')
+            - A ModuleType object representing an already imported module
+        recursive (bool, optional): Whether to recursively import submodules of
+            subpackages. Defaults to True. If False, only direct submodules of the
+            given package are imported.
+
+    Returns:
+        dict[str, ModuleType]: A dictionary mapping module names to their corresponding
+            imported module objects. Keys are fully qualified module names (strings)
+            and values are the imported ModuleType objects. If a module fails to import
+            due to ModuleNotFoundError, it is silently skipped and not included in the
+            returned dictionary.
+    """
+    if isinstance(package, str):
+        package = importlib.import_module(package)
+    results = {}
+    for _, name, is_pkg in pkgutil.walk_packages(package.__path__):
+        full_name = package.__name__ + "." + name
+        try:
+            results[full_name] = importlib.import_module(full_name)
+        except ModuleNotFoundError:
+            continue
+        if recursive and is_pkg:
+            results.update(import_submodules(full_name))
+    return results
+
+
+def import_testcases() -> None:
+    """Import all testcases in this module"""
+    import_submodules(__name__)

operator-pipeline-images/operatorcert/integration/testcases/basic.py

@@ -0,0 +1,82 @@
+"""
+Basic test cases
+"""
+
+import logging
+from pathlib import Path
+from shutil import rmtree
+from tempfile import mkdtemp
+from typing import Optional
+
+from git import Repo
+
+from operatorcert.integration.config import Config
+from operatorcert.integration.testcase import BaseTestCase, integration_test_case
+
+
+@integration_test_case
+class AddBundle(BaseTestCase):  # pragma: no cover
+    """
+    Test the addition of a non-FBC bundle
+    """
+
+    def __init__(self, config: Config, logger: logging.Logger) -> None:
+        super().__init__(config, logger)
+        self.tempdir: Optional[Path] = None
+
+    def setup(self) -> None:
+        self.tempdir = Path(mkdtemp(prefix=self.run_id + "-"))
+        test_branch = "add-bundle"
+        remote_branch = f"test-{self.run_id}"
+        git_fixture_dir = self.tempdir / "git-fixture"
+        # clone fixture repo locally
+        local_repo = Repo.clone_from(
+            self.config.fixtures_repository.url, git_fixture_dir, branch=test_branch
+        )
+        # configure remotes
+        local_repo.create_remote("operators", url=self.config.operator_repository.url)
+        local_repo.create_remote(
+            "contributor", url=self.config.contributor_repository.url
+        )
+        # The fixture branch contains the operator structure needed for the test
+        # and the latest commit is the change being tested. This means we need
+        # to exclude the latest commit when pushing to the operator repo.
+        local_repo.create_head(
+            # create local contributor branch
+            "contributor",
+            commit="HEAD",
+        )
+        local_repo.remotes.contributor.push(
+            # force push local contributor branch to test branch in contributor repo
+            f"+contributor:{remote_branch}"
+        )
+        local_repo.create_head(
+            # create local operators branch discarding latest commit
+            "operators",
+            commit="HEAD~1",
+        )
+        local_repo.remotes.operators.push(
+            # force push local operators branch to test branch in operators repo
+            f"+operators:{remote_branch}"
+        )
+        # TODO: create github webhook
+        # TODO: create github PR
+
+    def watch(self) -> None:
+        # TODO: wait until pipelinerun finishes
+        pass
+
+    def validate(self) -> None:
+        # TODO: check pipelinerun status
+        # TODO: check generated bundle image
+        # TODO: check bundle is in index
+        pass
+
+    def cleanup(self) -> None:
+        # TODO: remove bundle image
+        # TODO: remove index image
+        # TODO: remove github webhook
+        # TODO: delete test git branches
+        # remove temp dir
+        if self.tempdir:
+            rmtree(self.tempdir, ignore_errors=True)

operator-pipeline-images/tests/integration/conftest.py

@@ -36,6 +36,9 @@ def operator_pipelines_path(tmp_path: Path) -> Path:
 def integration_tests_config_file(tmp_path: Path) -> Path:
     sample_config = """
 ---
+fixtures_repository:
+    url: https://github.com/foo/foo
+    token: secretABC
 operator_repository:
     url: https://github.com/foo/bar
     token: asdfg