feat: Add labels and karmada kubeconfig merge

Signed-off-by: Panos Mavrikos <pmavrikos@nubificus.co.uk>

Author: panosmaurikos

orchestrators/inventory.yml

@@ -2,52 +2,110 @@ all:
   children:
     management_cluster:
       hosts:
-        mls00:
-          ansible_host: 192.168.5.79
+        testvm00:
+          ansible_host: 192.168.5.12
           ansible_user: mlsysops
           ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
           ansible_python_interpreter: /usr/bin/python3
           k3s_cluster_name: management
           pod_cidr: "10.10.0.0/16"
           service_cidr: "10.11.0.0/16"
+          labels:
+            is_vm: true
+            continuumLayer: continuum
+            vaccel: "false"
 
     cluster1:
       children:
         master_nodes:
           hosts:
-            mls01:
-              ansible_host: 192.168.5.25
+            testvm01:
+              ansible_host: 192.168.5.55
               ansible_user: mlsysops
               ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
               ansible_python_interpreter: /usr/bin/python3
               k3s_cluster_name: cluster1
               pod_cidr: "10.12.0.0/16"
               service_cidr: "10.13.0.0/16"
+              labels:
+                is_vm: true
+                continuumLayer: cluster
+                vaccel: "false"
         worker_nodes:
           hosts:
-            mls02:
-              ansible_host: 192.168.5.55
+            testvm02:
+              ansible_host: 192.168.5.7
               ansible_user: mlsysops
               ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
               ansible_python_interpreter: /usr/bin/python3
               k3s_cluster_name: cluster1
-            mls03:
-              ansible_host: 192.168.5.80
+              labels:
+                is_vm: true
+                continuumLayer: node
+                vaccel: "false"
+            testvm03:
+              ansible_host: 192.168.5.79
               ansible_user: mlsysops
               ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
               ansible_python_interpreter: /usr/bin/python3
               k3s_cluster_name: cluster1
-#    cluster2:
- #     children:
-  #      master_nodes:
-   #       hosts:
-    #        mls04:
-     #         ansible_host: 192.168.5.56
-      #        ansible_user: mlsysops
-       #       ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
-        #      ansible_python_interpreter: /usr/bin/python3
-         #     k3s_cluster_name: cluster2
-          #    pod_cidr: "10.14.0.0/16"
-           #   service_cidr: "10.15.0.0/16"
-        #worker_nodes:
-         # hosts: {}
+              labels:
+                is_vm: true
+                continuumLayer: node
+                vaccel: "false"
+    cluster2:
+      children:
+        master_nodes:
+          hosts:
+            testvm04:
+              ansible_host: 192.168.5.41
+              ansible_user: mlsysops
+              ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
+              ansible_python_interpreter: /usr/bin/python3
+              k3s_cluster_name: cluster2
+              pod_cidr: "10.14.0.0/16"
+              service_cidr: "10.15.0.0/16"
+              labels:
+                is_vm: true
+                continuumLayer: cluster
+                vaccel: "false"
+        worker_nodes:
+          hosts: 
+            testvm05:
+              ansible_host: 192.168.5.29
+              ansible_user: mlsysops
+              ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
+              ansible_python_interpreter: /usr/bin/python3
+              k3s_cluster_name: cluster2
+              labels:
+                is_vm: true
+                continuumLayer: node
+                vaccel: "false"
+    cluster3:
+      children:
+        master_nodes:
+          hosts:
+            testvm06:
+              ansible_host: 192.168.5.9
+              ansible_user: mlsysops
+              ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
+              ansible_python_interpreter: /usr/bin/python3
+              k3s_cluster_name: test1
+              pod_cidr: "10.14.0.0/16"
+              service_cidr: "10.15.0.0/16"
+              labels:
+                is_vm: true
+                continuumLayer: cluster
+                vaccel: "false"
+        worker_nodes:
+          hosts: 
+            testvm07:
+              ansible_host: 192.168.5.63
+              ansible_user: mlsysops
+              ansible_ssh_private_key_file: /home/mlsysops/.ssh/id_rsa
+              ansible_python_interpreter: /usr/bin/python3
+              k3s_cluster_name: test1
+              labels:
+                is_vm: true
+                continuumLayer: node
+                vaccel: "false"  

orchestrators/k3s-install.yml

@@ -21,11 +21,15 @@
         update_cache: true
 
     - name: Install k3s on master nodes
-      shell: >
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION={{ k3s_version }} sh -s - server
-        --cluster-cidr={{ hostvars[inventory_hostname].pod_cidr }}
-        --service-cidr={{ hostvars[inventory_hostname].service_cidr }}
-        --node-ip {{ ansible_host }}
+      shell: |
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION={{ k3s_version }} sh -s - server \
+          --cluster-cidr={{ hostvars[inventory_hostname].pod_cidr }} \
+          --service-cidr={{ hostvars[inventory_hostname].service_cidr }} \
+          --node-ip {{ ansible_host }} \
+          {%- for item in labels | dict2items %}
+          --node-label {{ item.key }}={{ item.value }} \
+          {%- endfor %}
+
       args:
       when: inventory_hostname in groups['master_nodes'] or 'management_cluster' in group_names
 
@@ -36,12 +40,14 @@
       when: inventory_hostname in groups['master_nodes'] or 'management_cluster' in group_names
 
     - name: Install k3s on worker nodes
-      shell: >
-        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION={{ k3s_version }} sh -s - agent
-        --server https://{{ hostvars[groups['master_nodes'][0]]['ansible_host'] }}:6443
-        --token {{ hostvars[groups['master_nodes'][0]]['node_token']['content'] | b64decode | trim }}
-        --node-ip {{ ansible_host }}
+      shell: |
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION={{ k3s_version }} sh -s - agent \
+        --server https://{{ (groups['master_nodes'] | map('extract', hostvars) | selectattr('k3s_cluster_name', 'eq', k3s_cluster_name) | first).ansible_host }}:6443 \
+        --token {{ (groups['master_nodes'] | map('extract', hostvars) | selectattr('k3s_cluster_name', 'eq', k3s_cluster_name) | first).node_token['content'] | b64decode | trim }} \
+        --node-ip {{ ansible_host }} \
+        {% for item in labels | dict2items %}--node-label {{ item.key }}={{ item.value }} {% endfor %}
       args:
+        executable: /bin/bash
       when: inventory_hostname in groups['worker_nodes']
 
     - name: Ensure k3s config is readable

orchestrators/karmada-install.yml

@@ -5,12 +5,14 @@
   become: yes
   become_user: root
   become_method: sudo
+  vars:
+    kubeconfig_dir: "/home/{{ ansible_user }}/kubeconfigs"
 
   tasks:
     - name: Fetch kubeconfig from cluster master nodes
       ansible.builtin.fetch:
         src: "/etc/rancher/k3s/k3s.yaml"
-        dest: "/tmp/{{ inventory_hostname }}-k3s.yaml"
+        dest: "{{kubeconfig_dir}}/{{ inventory_hostname }}-k3s.yaml"
         flat: yes
       when: inventory_hostname in groups['master_nodes']
 
@@ -20,25 +22,29 @@
   become_user: root
   become_method: sudo
   gather_facts: no
+  vars:
+    kubeconfig_dir: "/home/{{ ansible_user }}/kubeconfigs"
 
   tasks:
     - name: Ensure temporary directory exists
       ansible.builtin.file:
-        path: /tmp/kubeconfigs
+        path: "{{ kubeconfig_dir }}"
         state: directory
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
         mode: '0755'
 
     - name: Copy fetched kubeconfig to management vm
       ansible.builtin.copy:
-        src: "/tmp/{{ item }}-k3s.yaml"
-        dest: "/tmp/kubeconfigs/{{ item }}-k3s.yaml"
+        src: "{{ kubeconfig_dir }}/{{ item }}-k3s.yaml"
+        dest: "{{ kubeconfig_dir }}/{{ item }}-k3s.yaml"
         mode: '0644'
       loop: "{{ groups['master_nodes'] }}"
 
     - name: Adjust kubeconfig server address for cluster master nodes
       ansible.builtin.replace:
-        path: "/tmp/kubeconfigs/{{ item }}-k3s.yaml"
-        regexp: '127.0.0.1'
+        path: "{{ kubeconfig_dir }}/{{ item }}-k3s.yaml"
+        regexp: '127\.0\.0\.1'
         replace: "{{ hostvars[item]['ansible_host'] }}"
       loop: "{{ groups['master_nodes'] }}"
 
@@ -51,11 +57,20 @@
     managementvm_ip: "{{ hostvars[inventory_hostname]['ansible_host'] }}"
     managementvm_hostname: "{{ inventory_hostname }}"
     karmada_apiserver_nodeport: 32443
+    kubeconfig_dir: "/home/{{ ansible_user }}/kubeconfigs"
 
   tasks:
     - name: Gathering Facts
       ansible.builtin.setup:
 
+    - name: Ensure working directory exists
+      file:
+        path: "/home/{{ ansible_user }}/mlsysopsansible"
+        state: directory
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+        mode: '0755'
+
     - name: Create values.yml
       ansible.builtin.copy:
         dest: "/home/{{ ansible_user }}/mlsysopsansible/values.yml"
@@ -200,6 +215,7 @@
     karmada_config: "/home/{{ ansible_user }}/karmada-kubeconfig.yaml"
     managementvm_ip: "{{ hostvars[inventory_hostname]['ansible_host'] }}"
     karmada_apiserver_nodeport: 32644
+    kubeconfig_dir: "/home/{{ ansible_user }}/kubeconfigs"
 
   tasks:
     - name: Patch Karmada API server service to NodePort
@@ -224,7 +240,7 @@
   become: yes
   vars:
     karmada_config: "/home/{{ ansible_user }}/karmada-kubeconfig.yaml"
-
+    kubeconfig_dir: "/home/{{ ansible_user }}/kubeconfigs"
   tasks:
     - name: Get list of all clusters
       ansible.builtin.set_fact:
@@ -258,7 +274,7 @@
       ansible.builtin.command: |
         karmadactl join {{ item }} \
           --kubeconfig {{ karmada_config }} \
-          --cluster-kubeconfig /tmp/kubeconfigs/{{ cluster_masters[item] }}-k3s.yaml
+          --cluster-kubeconfig {{ kubeconfig_dir }}/{{ cluster_masters[item] }}-k3s.yaml
       loop: "{{ cluster_list }}"
       register: join_clusters
       failed_when: join_clusters.rc != 0
@@ -274,3 +290,79 @@
     - name: Debug Karmada clusters
       debug:
         var: karmada_clusters.stdout
+
+- name: Merge kubeconfig files into Karmada config
+  hosts: management_cluster
+  become: yes
+  become_user: root
+  become_method: sudo
+  vars:
+    karmada_config: "/home/{{ ansible_user }}/karmada-kubeconfig.yaml"
+    kubeconfig_dir: "/home/{{ ansible_user }}/kubeconfigs"
+    temp_dir: "/tmp/kubeconfigs"
+
+  tasks:
+    - name: Ensure temporary directory exists
+      ansible.builtin.file:
+        path: "{{ temp_dir }}"
+        state: directory
+        mode: '0755'
+
+    - name: Find all cluster kubeconfig files
+      ansible.builtin.find:
+        paths: "{{ kubeconfig_dir }}"
+        patterns: "*-k3s.yaml"
+      register: found_configs
+
+    - name: Copy kubeconfigs to temporary directory
+      ansible.builtin.copy:
+        src: "{{ item.path }}"
+        dest: "{{ temp_dir }}/{{ item.path | basename }}"
+        remote_src: yes
+        mode: '0644'
+      loop: "{{ found_configs.files }}"
+      when: found_configs.files | length > 0
+
+    - name: Normalize context and cluster names in temporary kubeconfigs
+      ansible.builtin.shell: |
+        name={{ item.path | basename | regex_replace('-k3s.yaml','') }}
+       
+        kubectl --kubeconfig={{ temp_dir }}/{{ item.path | basename }} config rename-context default ${name} || true
+       
+        sed -i "s/name: default/name: ${name}/" {{ temp_dir }}/{{ item.path | basename }}
+        sed -i "s/cluster: default/cluster: ${name}/" {{ temp_dir }}/{{ item.path | basename }}
+      args:
+        executable: /bin/bash
+      loop: "{{ found_configs.files }}"
+      when: found_configs.files | length > 0
+
+    - name: Merge all kubeconfigs into Karmada config
+      ansible.builtin.shell: |
+        export KUBECONFIG=$(find {{ temp_dir }} -name '*-k3s.yaml' | tr '\n' ':' | sed 's/:$//'):{{ karmada_config }}
+        kubectl config view --flatten > /tmp/merged-karmada.yaml
+      args:
+        executable: /bin/bash
+      when: found_configs.files | length > 0
+
+    - name: Validate merged kubeconfig
+      ansible.builtin.shell: |
+        kubectl --kubeconfig=/tmp/merged-karmada.yaml config view
+      register: validate_result
+      failed_when: validate_result.rc != 0
+      when: found_configs.files | length > 0
+
+    - name: Replace original Karmada config with merged one
+      ansible.builtin.copy:
+        src: "/tmp/merged-karmada.yaml"
+        remote_src: yes
+        dest: "{{ karmada_config }}"
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+        mode: '0600'
+      when: found_configs.files | length > 0 and validate_result.rc == 0
+
+    - name: Clean up temporary files and directory
+      ansible.builtin.file:
+        path: "{{ temp_dir }}"
+        state: absent
+      when: found_configs.files | length > 0