Merge pull request #1377 from projectsyn/feat/catalog-compile-processes

Introduce new parameter `--processes` for `catalog compile`

Author: simu

commodore/cli/catalog.py

@@ -164,6 +164,16 @@ def _complete_clusters(ctx: click.Context, _, incomplete: str) -> list[str]:
     + "uncommitted changes in tracked files count as local changes."
     + "The parameter has no effect if `--local` is given.",
 )
+@click.option(
+    "--processes",
+    type=int,
+    default=0,
+    show_default=True,
+    help="Control the number of worker processes that are spawned when compiling the catalog. "
+    + "A value of `0` will set the number of worker processes to the number of CPUs available "
+    + "on the system. Note that this parameter doesn't adjust the number of threads used by "
+    + "reclass-rs.",
+)
 @options.verbosity
 @options.pass_config
 # pylint: disable=too-many-arguments
@@ -187,6 +197,7 @@ def compile_catalog(
     migration,
     dynamic_fact: str,
     force: bool,
+    processes: int,
 ):
     config.update_verbosity(verbose)
     config.api_url = api_url
@@ -204,6 +215,7 @@ def compile_catalog(
     config.fetch_dependencies = fetch_dependencies
     config.dynamic_facts = parse_dynamic_facts_from_cli(dynamic_fact)
     config.force = not config.local and force
+    config.processes = processes
 
     if config.push and (
         config.global_repo_revision_override or config.tenant_repo_revision_override

commodore/config.py

@@ -103,6 +103,7 @@ class Config:
     _request_timeout: int
     _managed_tools: dict[str, str]
     _api_token: Optional[str]
+    _processes: int
 
     oidc_client: Optional[str]
     oidc_discovery_url: Optional[str]
@@ -144,6 +145,7 @@ def __init__(
         self._github_token = None
         self._request_timeout = 5
         self._managed_tools = {}
+        self._processes = 0
 
     @property
     def verbose(self):
@@ -327,6 +329,14 @@ def managed_tools(self) -> dict[str, str]:
     def managed_tools(self, managed_tools: dict[str, str]):
         self._managed_tools = managed_tools
 
+    @property
+    def processes(self) -> int:
+        return self._processes
+
+    @processes.setter
+    def processes(self, processes: int):
+        self._processes = processes
+
     @property
     def inventory(self):
         return self._inventory

commodore/helpers.py

@@ -251,6 +251,10 @@ def kapitan_compile(
     refController = RefController(config.refs_dir)
     if fake_refs:
         refController.register_backend(FakeVaultBackend())
+    processes: Optional[int] = config.processes
+    if config.processes == 0:
+        processes = cpu_count()
+
     click.secho("Compiling catalog...", bold=True)
     # workaround the non-modifiable Namespace() default value for cached.args
     cached.args.inventory_backend = "reclass-rs"
@@ -261,7 +265,7 @@ def kapitan_compile(
     cached.args.verbose = config.trace
     cached.args.output_path = output_dir
     cached.args.targets = targets
-    cached.args.parallelism = cpu_count()
+    cached.args.parallelism = processes
     cached.args.labels = None
     cached.args.prune = False
     cached.args.indent = 2

docs/modules/ROOT/pages/reference/cli.adoc

@@ -129,6 +129,12 @@ With `--no-force`, local changes in global, tenant, or dependency checkouts are
 Specifying `--force` has no effect if `--local` is given, and is silently ignored.
 Defaults to `--no-force`.
 
+*--processes*::
+  Control the number of worker processes that are spawned when compiling the catalog.
+  A value of `0` will set the number of worker processes to the number of CPUs available on the system.
+  Note that this parameter doesn't adjust the number of threads used by reclass-rs.
+  Defaults to `0`.
+
 *--help*::
   Show catalog clean usage and options then exit.