Merge pull request #1174 from projectsyn/feat/external-tool-mgmt

Add commands to manage required external tools

Author: simu

.github/workflows/test.yml

@@ -163,8 +163,4 @@ jobs:
       run: make docker
     - name: Run image
       run: |
-        docker run docker.io/projectsyn/commodore:test || exit_code=$?
-        if [ "$exit_code" -ne 2 ]; then
-          echo "Unexpected exit code $exit_code, expected 2"
-          exit $exit_code
-        fi
+        docker run docker.io/projectsyn/commodore:test version

Dockerfile

@@ -15,6 +15,7 @@ ARG POETRY_VERSION=1.8.5
 RUN apt-get update && apt-get install -y --no-install-recommends \
       build-essential \
       curl \
+      git \
       libffi-dev \
  && rm -rf /var/lib/apt/lists/* \
  && curl -sSL https://install.python-poetry.org | python - --version ${POETRY_VERSION} \
@@ -37,22 +38,13 @@ RUN sed -i "s/^__git_version__.*$/__git_version__ = '${GITVERSION}'/" commodore/
 
 RUN pip install ./dist/syn_commodore-*-py3-none-any.whl
 
-RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \
- && chmod 700 get_helm.sh \
- && ./get_helm.sh \
- && mv /usr/local/bin/helm /usr/local/bin/helm3 \
- && curl -LO https://git.io/get_helm.sh \
- && chmod 700 get_helm.sh \
- && ./get_helm.sh \
- && mv /usr/local/bin/helm /usr/local/bin/helm2
-
 ARG KUSTOMIZE_VERSION=5.7.0
 ARG JSONNET_BUNDLER_VERSION=v0.6.3
+ARG HELM_VERSION=v3.18.4
 
-RUN ./tools/install-jb.sh ${JSONNET_BUNDLER_VERSION} \
- && curl -fsSLO "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" \
- && chmod +x install_kustomize.sh \
- && ./install_kustomize.sh ${KUSTOMIZE_VERSION} /usr/local/bin
+RUN commodore tool install helm --version ${HELM_VERSION} \
+ && commodore tool install kustomize --version ${KUSTOMIZE_VERSION} \
+ && commodore tool install jb --version ${JSONNET_BUNDLER_VERSION}
 
 FROM base AS runtime
 
@@ -73,12 +65,17 @@ COPY --from=builder \
 COPY --from=builder \
       /usr/local/bin/kapitan* \
       /usr/local/bin/commodore* \
-      /usr/local/bin/helm* \
-      /usr/local/bin/jb \
-      /usr/local/bin/kustomize \
       /usr/local/bin/
 
-RUN ln -s /usr/local/bin/helm3 /usr/local/bin/helm
+COPY --from=builder \
+      /app/.cache/commodore/tools/ \
+      /app/.cache/commodore/tools/
+
+RUN ln -s \
+      /app/.cache/commodore/tools/helm \
+      /app/.cache/commodore/tools/jb \
+      /app/.cache/commodore/tools/kustomize \
+      /usr/local/bin/
 
 COPY ./tools/entrypoint.sh /usr/local/bin/
 

commodore/cli/__init__.py

@@ -9,7 +9,8 @@
 from reclass_rs import Reclass
 
 from dotenv import load_dotenv, find_dotenv
-from commodore import __git_version__, __version__
+
+from commodore import __git_version__, __version__, tools
 from commodore.config import Config
 from commodore.version import version_info
 
@@ -20,6 +21,7 @@
 from .inventory import inventory_group
 from .package import package_group
 from .oidc import commodore_fetch_token, commodore_login
+from .tool import tool_group
 
 
 def _version():
@@ -75,6 +77,7 @@ def commodore(ctx, working_dir, verbose, request_timeout):
 commodore.add_command(component_group)
 commodore.add_command(inventory_group)
 commodore.add_command(package_group)
+commodore.add_command(tool_group)
 commodore.add_command(commodore_login)
 commodore.add_command(commodore_fetch_token)
 commodore.add_command(commodore_version)
@@ -83,6 +86,7 @@ def commodore(ctx, working_dir, verbose, request_timeout):
 def main():
     multiprocessing.set_start_method("spawn")
     Reclass.set_thread_count(0)
+    tools.setup_path()
 
     load_dotenv(dotenv_path=find_dotenv(usecwd=True))
     commodore.main(

commodore/cli/tool.py

@@ -0,0 +1,176 @@
+from typing import Optional
+
+import click
+
+import commodore.cli.options as options
+
+from commodore import tools
+from commodore.config import Config
+
+
+@click.group(
+    name="tool",
+    short_help="Manage required external tools",
+)
+@options.verbosity
+@options.pass_config
+@options.github_token
+def tool_group(config: Config, verbose: int, github_token: str):
+    """Commands to manage required external tools.
+
+    Currently, `helm`, jsonnet-bundler/`jb`, and `kustomize` are required
+    external tools.
+    """
+    config.update_verbosity(verbose)
+    config.managed_tools = tools.load_state()
+    config.github_token = github_token
+
+
+@tool_group.command(name="list", short_help="List external tools")
+@options.verbosity
+@options.pass_config
+@options.github_token
+@click.option(
+    "--version-check/--skip-version-check",
+    " / -V",
+    default=True,
+    help="Query GitHub API to get latest version",
+)
+def tool_list(config: Config, verbose: int, github_token: str, version_check: bool):
+    """List the version, location and management state of the required external tools.
+
+    Currently, `helm`, jsonnet-bundler/`jb`, and `kustomize` are required
+    external tools. By default, the command also queries the GitHub API to
+    determine the latest available version of each tool and indicates whether an
+    update is available.
+
+    Optionally, the command accepts a GitHub personal access token (PAT) to
+    avoid running into the fairly strict unauthenticated GitHub rate limits.
+    """
+    config.update_verbosity(verbose)
+    config.github_token = github_token
+    tools.list_tools(config, version_check)
+
+
+@tool_group.command(name="install", short_help="Install external tools")
+@options.verbosity
+@options.pass_config
+@options.github_token
+@click.option(
+    "--version",
+    default=None,
+    metavar="VERSION",
+    help="A version to install for the requested tool. "
+    + "By default, the latest version is installed.",
+)
+@click.option(
+    "--missing",
+    is_flag=True,
+    default=False,
+    help="Install the latest version for all currently unmanaged tools. "
+    + "This flag and providing a tool name on the command line are mutually exclusive.",
+)
+@click.argument("tool", required=False, default="")
+def tool_install(
+    config: Config,
+    verbose: int,
+    tool: str,
+    version: Optional[str],
+    github_token: str,
+    missing: bool,
+):
+    """Install one of the required tools in `$XDG_CACHE_DIR/commodore/tools`.
+
+    The command will fail for tools which are already managed by Commodore.
+
+    By default, the command will install the latest available tool version. For
+    `helm` and `kustomize`, the command downloads the official installation
+    scripts and executes them with appropriate arguments. For `jb`, the command
+    directly downloads the requested version from the GitHub release page.
+
+    Optionally, the command accepts a tool version to install. The command
+    accepts versions prefixed  with "v" and unprefixed versions.
+
+    Alternatively, the command can install the latest version for all required
+    tools which aren't managed by Commodore yet by passing flag `--missing`.
+    This flag is mutually exclusive with providing a tool name.
+    """
+    config.update_verbosity(verbose)
+    config.github_token = github_token
+
+    if not tool and not missing or tool and missing:
+        raise click.ClickException(
+            "`commodore tool install` expects to be called with either a tool name or the `--missing` flag."
+        )
+    if missing and version:
+        click.secho(
+            "Flag `--version` has no effect when calling the command with `--missing`.",
+            fg="yellow",
+        )
+
+    if missing:
+        tools.install_missing_tools(config)
+    else:
+        tools.install_tool(config, tool, version)
+
+
+@tool_group.command(name="upgrade", short_help="Upgrade external tools")
+@options.verbosity
+@options.github_token
+@options.pass_config
+@click.option(
+    "--version",
+    default=None,
+    metavar="VERSION",
+    help="A version to upgrade (or downgrade) to for the requested tool. "
+    + "By default, the tool is upgraded to the latest version.",
+)
+@click.option(
+    "--all",
+    is_flag=True,
+    default=False,
+    help="Upgrade all currently managed tools to their latest versions. "
+    + "This flag and providing a tool name on the command line are mutually exclusive.",
+)
+@click.argument("tool", required=False, default="")
+def tool_upgrade(
+    config: Config,
+    verbose: int,
+    tool: str,
+    version: Optional[str],
+    github_token: str,
+    all: bool,
+):
+    """Upgrade (or downgrade) one of the required tools in `$XDG_CACHE_DIR/commodore/tools`.
+
+    The command will fail for tools which aren't managed by Commodore yet.
+
+    By default, the command will upgrade the tool to the latest available
+    version. For `helm` and `kustomize`, the command downloads the official
+    installation scripts and executes them with appropriate arguments. For `jb`,
+    the command directly downloads the requested version from the GitHub release
+    page.
+
+    Optionally, the command accepts a tool version to upgrade (or downgrade) to.
+    The command accepts versions prefixed with "v" and unprefixed versions.
+
+    Alternatively, the command can upgrade all managed tools to their latest
+    versions by passing flag `--all`. This flag is mutually exclusive with
+    providing a tool name.
+    """
+    config.update_verbosity(verbose)
+    config.github_token = github_token
+
+    if not tool and not all or tool and all:
+        raise click.ClickException(
+            "`commodore tool upgrade` expects to be called with either a tool name or the `--all` flag."
+        )
+    if all and version:
+        click.secho(
+            "Flag `--version` has no effect when calling the command with `--all`.",
+            fg="yellow",
+        )
+    if all:
+        tools.upgrade_all_tools(config)
+    else:
+        tools.upgrade_tool(config, tool, version)

commodore/config.py

@@ -101,6 +101,7 @@ class Config:
     _dynamic_facts: dict[str, Any]
     _github_token: Optional[str]
     _request_timeout: int
+    _managed_tools: dict[str, str]
 
     oidc_client: Optional[str]
     oidc_discovery_url: Optional[str]
@@ -141,6 +142,7 @@ def __init__(
         self._dynamic_facts = {}
         self._github_token = None
         self._request_timeout = 5
+        self._managed_tools = {}
 
     @property
     def verbose(self):
@@ -316,6 +318,14 @@ def request_timeout(self) -> int:
     def request_timeout(self, timeout: int):
         self._request_timeout = timeout
 
+    @property
+    def managed_tools(self) -> dict[str, str]:
+        return self._managed_tools
+
+    @managed_tools.setter
+    def managed_tools(self, managed_tools: dict[str, str]):
+        self._managed_tools = managed_tools
+
     @property
     def inventory(self):
         return self._inventory