feat: create descriptions from inventory.yaml. remove nb-api. add github action

Signed-off-by: Alexandros Patras <patras@uth.gr>

Author: alexanpatr

.github/workflows/build-cli-pkg.yml

@@ -0,0 +1,73 @@
+name: Build & publish
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ["base-dind-2204-amd64"]
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Find SHA
+      run: |
+        if [[ "${{github.event.pull_request.head.sha}}" != "" ]]
+        then
+          echo "ARTIFACT_SHA=$(echo ${{github.event.pull_request.head.sha}})" >> $GITHUB_ENV
+          echo "CI_COMMIT_TAG=0.0.0-${{ github.run_id }}" >> $GITHUB_ENV
+        else
+          echo "ARTIFACT_SHA=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV
+          echo "CI_COMMIT_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
+        fi
+
+    - name: Install python dependencies
+      run: |
+        sudo apt update && sudo apt install -y python3.10-venv
+        python3 -m venv .venv
+        . .venv/bin/activate
+        .venv/bin/python3 -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then .venv/bin/python3 -m pip install -r requirements.txt; fi
+
+    #- name: Lint with flake8
+    #  run: |
+    #    # stop the build if there are Python syntax errors or undefined names
+    #    python3 -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+    #    # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+    #    python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+    #- name: Test with pytest
+    #  env:
+    #    PYTHONPATH: .
+    #  run: |
+    #    python3 -m build
+    #    python3 -m pytest
+
+    #- name: Test with coverage
+    #  env:
+    #    PYTHONPATH: .
+    #  run: |
+    #    python3 -m pytest tests/
+
+    - name: Build pip package
+      run: |
+        . .venv/bin/activate
+        .venv/bin/python3 -m pip install build
+        cd agents
+        ../.venv/bin/python3 -m build
+        VERSION=$(python3 setup.py --version)
+        PYVER=$(python3 --version | awk '{print $2}' | awk -F\. '{print $1 $2 }')
+
+    # FIXME: add pypi publish step
+    - name: Publish package
+      #if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_TOKEN }}
+        packages-dir: agents/dist

.github/workflows/ci.yml

@@ -35,6 +35,15 @@ jobs:
     uses: ./.github/workflows/build-mlsysops-pkg.yml
     secrets: inherit
 
+  build-cli-pkg:
+    #needs: [validate-files-and-commits, lint]
+    name: Build mlsysops CLI pkg
+    if: | 
+      contains(github.event.pull_request.labels.*.name, 'ok-to-test') &&
+      !contains(github.event.pull_request.labels.*.name, 'skip-build-pkg')
+    uses: ./.github/workflows/build-cli-pkg.yml
+    secrets: inherit
+
   build-agent-containers:
     needs: [build-agent-pkg]
     name: Build containers

mlsysops-cli/mlsysops_cli/__init__.py

@@ -1,5 +1,4 @@
 # mlsysops/__init__.py
 
 from .cli import cli
-
-__version__ = "0.1.0"
+from .deployment import *

mlsysops-cli/mlsysops_cli/cli.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import traceback
 
 import click
 import requests
@@ -13,6 +14,8 @@
     deploy_node_agents,
 )
 
+from mlsysops_cli.deployment.descriptions_util import create_app_yaml
+
 # Configurable IP and PORT via environment variables
 IP = os.getenv("MLS_API_IP", "127.0.0.1")
 PORT = os.getenv("MLS_API_PORT", "8000")
@@ -313,9 +316,15 @@ def framework():
 
 
 @click.command(help="Deploy all components (core services, continuum, clusters, nodes)")
-def deploy_all():
+@click.option('--path', type=click.Path(exists=True), required=False, help='Path to the desriptions directory. It MUST include path/continuum,path/cluster,path/node')
+@click.option('--inventory', type=click.Path(exists=True), required=False, help='Path to the inventory YAML that was used from cluster/karmada setup ansible script.')
+def deploy_all(path, inventory):
+    # Ensure only one of the --path or --uri options is provided
+    if path and inventory:
+        click.secho('❌ Error: Provide only --path or --uri.')
+        return
     try:
-        run_deploy_all()
+        run_deploy_all(path, inventory)
     except Exception as e:
         click.secho(f"❌ Error during full deployment: {e}", fg='red')
 
@@ -329,35 +338,62 @@ def deploy_services():
 
 
 @click.command(help="Deploy the continuum agent")
-def deploy_continuum():
+@click.option('--path', type=click.Path(exists=True), required=False, help='Path to the desriptions directory. It MUST include path/continuum,path/cluster,path/node')
+@click.option('--inventory', type=click.Path(exists=True), required=False, help='Path to the inventory YAML that was used from cluster/karmada setup ansible script.')
+def deploy_continuum(path, inventory):
+    # Ensure only one of the --path or --uri options is provided
+    if path and inventory:
+        click.secho('❌ Error: Provide only --path or --uri.')
+        return
     try:
-        deploy_continuum_agents()
+        deploy_continuum_agents(path, inventory)
     except Exception as e:
         click.secho(f"❌ Error during continuum agent deployment: {e}", fg='red')
 
 
 @click.command(help="Deploy the cluster agents")
-def deploy_cluster():
+@click.option('--path', type=click.Path(exists=True), required=False, help='Path to the desriptions directory. It MUST include path/continuum,path/cluster,path/node')
+@click.option('--inventory', type=click.Path(exists=True), required=False, help='Path to the inventory YAML that was used from cluster/karmada setup ansible script.')
+def deploy_cluster(path, inventory):
+    # Ensure only one of the --path or --uri options is provided
+    if path and inventory:
+        click.secho('❌ Error: Provide only --path or --uri.')
+        return
     try:
-        deploy_cluster_agents()
+        deploy_cluster_agents(path, inventory)
     except Exception as e:
         click.secho(f"❌ Error during cluster agents deployment: {e}", fg='red')
 
 
 @click.command(help="Deploy the node agents")
-def deploy_node():
+@click.option('--path', type=click.Path(exists=True), required=False, help='Path to the desriptions directory. It MUST include path/continuum,path/cluster,path/node')
+@click.option('--inventory', type=click.Path(exists=True), required=False, help='Path to the inventory YAML that was used from cluster/karmada setup ansible script.')
+def deploy_node(path, inventory):
+    # Ensure only one of the --path or --uri options is provided
+    if path and inventory:
+        click.secho('❌ Error: Provide only --path or --uri.')
+        return
+
     try:
-        deploy_node_agents()
+        deploy_node_agents(path, inventory)
     except Exception as e:
         click.secho(f"❌ Error during node agents deployment: {e}", fg='red')
 
+@click.command(help="Create a test application description using an inventory YAML.")
+@click.option('--inventory', type=click.Path(exists=True), required=True, help='Path to the inventory YAML that was used from cluster/karmada setup ansible script.')
+def create_test_app_description(inventory):
+    try:
+        create_app_yaml(inventory)
+    except Exception as e:
+        click.secho(f"❌ Error during test application descriptions creation: {e}", fg='red')
 
-# Agrega los comandos al grupo 'framework'
+# Add commands to the 'framework' group
 framework.add_command(deploy_all)
 framework.add_command(deploy_services)
 framework.add_command(deploy_continuum)
 framework.add_command(deploy_cluster)
 framework.add_command(deploy_node)
+framework.add_command(create_test_app_description)
 
 cli.add_command(framework)
 

mlsysops-cli/mlsysops_cli/deployment/__init__.py

@@ -0,0 +1 @@
+from .descriptions_util import *

mlsysops-cli/mlsysops_cli/deployment/deploy.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from importlib.resources import files
 import kubernetes.client.rest
+import yaml
 from jinja2 import Environment, FileSystemLoader
 from kubernetes import client, config
 from kubernetes.client import ApiException
@@ -9,6 +10,7 @@
 from jinja2 import Template
 import subprocess
 from mlsysops_cli import deployment
+from mlsysops_cli.deployment.descriptions_util import create_cluster_yaml, create_worker_node_yaml,create_continuum_yaml
 
 
 def parse_yaml_from_file(path_obj: Path, template_variables: dict = {}) -> list | None:
@@ -264,10 +266,10 @@ def create_configmap_from_file(self, descriptions_directory, namespace, name, su
         Create a ConfigMap from all YAML files inside a given folder inside the package.
         """
         files_data_object = {}
-
+        directory = Path(descriptions_directory)
         try:
             for suffix in suffixes:
-                for file in descriptions_directory.glob(suffix):
+                for file in directory.glob(suffix):
                     print(f"Reading file: {file.name}")
                     file_data = file.read_text()
                     files_data_object[file.name] = file_data
@@ -485,13 +487,13 @@ def _check_required_env_vars(*required_vars):
     if missing:
         raise EnvironmentError(f"Missing required environment variables: {', '.join(missing)}")
 
-def run_deploy_all():
+def run_deploy_all(path, inventory_path):
     try:
         print("🚀 Deploying all MLSysOps components...")
         deploy_core_services()
-        deploy_continuum_agents()
-        deploy_cluster_agents()
-        deploy_node_agents()
+        deploy_continuum_agents(path, inventory_path)
+        deploy_cluster_agents(path, inventory_path)
+        deploy_node_agents(path, inventory_path)
         print("✅ All components deployed successfully.")
     except Exception as e:
         print(f"❌ Error in deploy_all: {e}")
@@ -518,15 +520,30 @@ def deploy_core_services():
     for r in parse_yaml_from_file(redis_path, {"KARMADA_HOST_IP": os.getenv("KARMADA_HOST_IP")}):
         client_k8s.create_or_update(r)
 
-def deploy_continuum_agents():
+def deploy_continuum_agents(path, inventory_path):
     print("🧠 Deploying Continuum Agent...")
     _check_required_env_vars("KARMADA_HOST_IP", "KARMADA_HOST_KUBECONFIG")
     client_k8s = KubernetesLibrary("apps", "v1", os.getenv("KARMADA_HOST_KUBECONFIG", "/etc/rancher/k3s/k3s.yaml"))
     _apply_namespace_and_rbac(client_k8s)
 
+    descriptions_path = os.getenv("CONTINUUM_SYSTEM_DESCRIPTIONS_PATH", "descriptions")
+    if path:
+        descriptions_path = path
+
+    if inventory_path:
+        # build the system descriptions
+        parent_dir = "descriptions"
+        descriptions_path = os.path.join(parent_dir, "continuum")
+        if not os.path.exists(parent_dir):
+            os.makedirs(parent_dir)
+        if not os.path.exists(descriptions_path):
+            os.makedirs(descriptions_path)
+
+
+        create_continuum_yaml(inventory_path, descriptions_path)
+
     # ConfigMap descriptions
-    desc_path = files(deployment).joinpath("descriptions/continuum")
-    client_k8s.create_configmap_from_file(desc_path, "mlsysops-framework", "continuum-system-description")
+    client_k8s.create_configmap_from_file(descriptions_path, "mlsysops-framework", "continuum-system-description")
 
     kubeconfig_path = files(deployment).joinpath("descriptions/continuum")
     client_k8s.create_configmap_from_file(kubeconfig_path, "mlsysops-framework", "continuum-karmadapi-config", suffixes=["*.kubeconfig"])
@@ -536,36 +553,81 @@ def deploy_continuum_agents():
     for r in parse_yaml_from_file(daemonset_path):
         client_k8s.create_or_update(r)
 
-def deploy_cluster_agents():
+def deploy_cluster_agents(path, inventory_path):
     print("🏢 Deploying Cluster Agents...")
     _check_required_env_vars("KARMADA_HOST_IP", "KARMADA_API_KUBECONFIG")
     client_karmada = KubernetesLibrary("apps", "v1", os.getenv("KARMADA_API_KUBECONFIG", "/etc/rancher/k3s/k3s.yaml"))
     _apply_namespace_and_rbac(client_karmada)
     client_karmada.apply_mlsysops_propagation_policies()
 
+    descriptions_path = os.getenv("CLUSTER_SYSTEM_DESCRIPTIONS_PATH", "descriptions")
+    if path:
+        descriptions_path = path
+
+    if inventory_path:
+        # build the system descriptions
+        parent_dir = "descriptions"
+        descriptions_path = os.path.join(parent_dir, "cluster")
+        if not os.path.exists(parent_dir):
+            os.makedirs(parent_dir)
+        if not os.path.exists(descriptions_path):
+            os.makedirs(descriptions_path)
+
+        with open(inventory_path, 'r') as file:
+            inventory = yaml.safe_load(file)
+
+        for cluster_name in inventory['all']['children']:
+            try:
+                print(f"Processing cluster: {cluster_name}")
+                create_cluster_yaml(inventory_path, cluster_name, descriptions_path)
+            except ValueError as e:
+                print(f"Skipping cluster '{cluster_name}': {e}")
+
     # ConfigMap
-    desc_path = files(deployment).joinpath("descriptions/cluster")
-    client_karmada.create_configmap_from_file(desc_path, "mlsysops-framework", "cluster-system-description")
+    client_karmada.create_configmap_from_file(descriptions_path, "mlsysops-framework", "cluster-system-description")
 
     # DaemonSet YAML
     daemonset_path = files(deployment).joinpath("cluster-agents-daemonset.yaml")
     for r in parse_yaml_from_file(daemonset_path, {"KARMADA_HOST_IP": os.getenv("KARMADA_HOST_IP")}):
         client_karmada.create_or_update(r)
 
-def deploy_node_agents():
+def deploy_node_agents(path, inventory_path):
     print("🧱 Deploying Node Agents...")
     _check_required_env_vars("KARMADA_HOST_IP", "KARMADA_API_KUBECONFIG")
     client_karmada = KubernetesLibrary("apps", "v1", os.getenv("KARMADA_API_KUBECONFIG", "/etc/rancher/k3s/k3s.yaml"))
 
+    descriptions_path = os.getenv("NODE_SYSTEM_DESCRIPTIONS_PATH","descriptions")
+    if path:
+      descriptions_path = path
+
+    if inventory_path:
+        # build the system descriptions
+        parent_dir = "descriptions"
+        descriptions_path = os.path.join(parent_dir,"nodes")
+        if not os.path.exists(parent_dir):
+            os.makedirs(parent_dir)
+        if not os.path.exists(descriptions_path):
+            os.makedirs(descriptions_path)
+
+        with open(inventory_path, 'r') as file:
+            inventory = yaml.safe_load(file)
+
+        for cluster_name in inventory['all']['children']:
+            try:
+                print(f"Processing cluster: {cluster_name}")
+                create_worker_node_yaml(inventory_path, cluster_name,descriptions_path)
+            except ValueError as e:
+                print(f"Skipping cluster '{cluster_name}': {e}")
+
     # ConfigMap
-    desc_path = files(deployment).joinpath("descriptions/node")
-    print(desc_path)
-    client_karmada.create_configmap_from_file(desc_path, "mlsysops-framework", "node-system-descriptions")
+    print(f"Using node systems decriptions from {descriptions_path}")
+    client_karmada.create_configmap_from_file(descriptions_path, "mlsysops-framework", "node-system-descriptions")
 
     # DaemonSet YAML
     daemonset_path = files(deployment).joinpath("node-agents-daemonset.yaml")
     for r in parse_yaml_from_file(daemonset_path, {"KARMADA_HOST_IP": os.getenv("KARMADA_HOST_IP")}):
         client_karmada.create_or_update(r)
+
 def _apply_namespace_and_rbac(client_instance):
     # Carga de namespace.yaml
     ns_path = files(deployment).joinpath("namespace.yaml")