test(slurm): support integration testing on private IP VMs via IAP tunnel (#5950)

Author: shubpal07

examples/hpc-enterprise-slurm.yaml

@@ -288,7 +288,7 @@ deployment_groups:
       # we recommend disabling public IPs if possible
       # but that requires your network to have a NAT or
       # private access configured
-      enable_login_public_ips: true
+      enable_login_public_ips: false
 
   - id: slurm_controller
     source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller
@@ -307,7 +307,7 @@ deployment_groups:
       # we recommend disabling public IPs if possible
       # but that requires your network to have a NAT or
       # private access configured
-      enable_controller_public_ips: true
+      enable_controller_public_ips: false
 
   - id: hpc_dashboard
     source: modules/monitoring/dashboard

examples/pfs-managed-lustre-slurm.yaml

@@ -89,7 +89,7 @@ deployment_groups:
     use: [network]
     settings:
       machine_type: n2-standard-4
-      enable_login_public_ips: true
+      enable_login_public_ips: false
       instance_image: $(vars.slurm_image)
 
   - id: slurm_controller
@@ -101,5 +101,5 @@ deployment_groups:
     - slurm_login
     settings:
       machine_type: n2-standard-4
-      enable_controller_public_ips: true
+      enable_controller_public_ips: false
       instance_image: $(vars.slurm_image)

tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml

@@ -79,53 +79,109 @@
       ansible.builtin.include_tasks:
         file: tasks/get_instance_ids.yml
 
-    - name: Get IP of a login node - Exact name provided
-      changed_when: false
-      register: get_login_ip
-      ansible.builtin.command: >-
-        gcloud compute instances describe --zone={{ zone }} {{ login_node }}
-        --format='get(networkInterfaces[0].accessConfigs[0].natIP)'
-      when: '"*" not in login_node'
-
-    - name: Set login_ip variable - Exact name provided
+    - name: Set login node name fact - Exact name provided
       ansible.builtin.set_fact:
-        login_ip: "{{ get_login_ip.stdout }}"
+        login_node_name: "{{ login_node }}"
       when: '"*" not in login_node'
 
-    - name: Get IP of a login node - Name pattern provided
+    - name: Get login node name - Name pattern provided
       changed_when: false
-      register: get_login_ip
+      register: get_login_name
       ansible.builtin.command: >-
-        gcloud compute instances list \
-          --format='get(networkInterfaces[0].accessConfigs[0].natIP)' --limit=1 \
-          --filter=NAME:{{ login_node }}
+        gcloud compute instances list
+        --format='value(name)' --limit=1
+        --filter=NAME:{{ login_node }}
       when: '"*" in login_node'
 
-    - name: Set login_ip variable - Name pattern provided
+    - name: Set login node name fact - Name pattern provided
       ansible.builtin.set_fact:
-        login_ip: "{{ get_login_ip.stdout }}"
+        login_node_name: "{{ get_login_name.stdout }}"
       when: '"*" in login_node'
 
+    - name: Check that login node name is resolved
+      ansible.builtin.assert:
+        that:
+        - login_node_name is defined
+        - login_node_name != ""
+
+    - name: Get Login Nat IP
+      changed_when: false
+      register: get_login_nat_ip
+      ansible.builtin.command: >-
+        gcloud compute instances describe --zone={{ zone }} {{ login_node_name }}
+        --format='value(networkInterfaces[0].accessConfigs[0].natIP)'
+      failed_when: false
+
+    - name: Get Login Network IP
+      changed_when: false
+      register: get_login_network_ip
+      ansible.builtin.command: >-
+        gcloud compute instances describe --zone={{ zone }} {{ login_node_name }}
+        --format='value(networkInterfaces[0].networkIP)'
+
+    - name: Set Login IP facts
+      ansible.builtin.set_fact:
+        login_ip: "{{ get_login_nat_ip.stdout | default(get_login_network_ip.stdout, true) }}"
+        login_is_private: "{{ get_login_nat_ip.stdout == '' }}"
+
     - name: Check that login IP is set
       ansible.builtin.assert:
         that:
         - login_ip is defined
         - login_ip != ""
 
-    - name: Print login public IP
+    - name: Print login IP info
       ansible.builtin.debug:
-        var: login_ip
+        msg: "Login IP: {{ login_ip }}, Private: {{ login_is_private }}"
+
+    - name: Set controller node name fact - Exact name provided
+      ansible.builtin.set_fact:
+        controller_node_name: "{{ controller_node }}"
+      when: '"*" not in controller_node'
 
-    - name: Get Controller IP
+    - name: Get controller node name - Name pattern provided
       changed_when: false
-      register: controller_ip
+      register: get_controller_name
       ansible.builtin.command: >-
-        gcloud compute instances describe --zone={{ zone }} {{ controller_node }}
-        --format='get(networkInterfaces[0].accessConfigs[0].natIP)'
+        gcloud compute instances list
+        --format='value(name)' --limit=1
+        --filter=NAME:{{ controller_node }}
+      when: '"*" in controller_node'
 
-    - name: Print controller public IP
+    - name: Set controller node name fact - Name pattern provided
+      ansible.builtin.set_fact:
+        controller_node_name: "{{ get_controller_name.stdout }}"
+      when: '"*" in controller_node'
+
+    - name: Check that controller node name is resolved
+      ansible.builtin.assert:
+        that:
+        - controller_node_name is defined
+        - controller_node_name != ""
+
+    - name: Get Controller Nat IP
+      changed_when: false
+      register: get_controller_nat_ip
+      ansible.builtin.command: >-
+        gcloud compute instances describe --zone={{ zone }} {{ controller_node_name }}
+        --format='value(networkInterfaces[0].accessConfigs[0].natIP)'
+      failed_when: false
+
+    - name: Get Controller Network IP
+      changed_when: false
+      register: get_controller_network_ip
+      ansible.builtin.command: >-
+        gcloud compute instances describe --zone={{ zone }} {{ controller_node_name }}
+        --format='value(networkInterfaces[0].networkIP)'
+
+    - name: Set Controller IP facts
+      ansible.builtin.set_fact:
+        controller_ip: "{{ get_controller_nat_ip.stdout | default(get_controller_network_ip.stdout, true) }}"
+        controller_is_private: "{{ get_controller_nat_ip.stdout == '' }}"
+
+    - name: Print controller IP info
       ansible.builtin.debug:
-        var: controller_ip.stdout_lines
+        msg: "Controller IP: {{ controller_ip }}, Private: {{ controller_is_private }}"
 
     ## Setup firewall for cloud build
     - name: Create firewall rule
@@ -171,14 +227,29 @@
 
     - name: Add Login node as host
       ansible.builtin.add_host:
-        hostname: "{{ login_ip }}"
+        name: login_node_host
+        ansible_host: "{{ login_node_name if login_is_private else login_ip }}"
+        ansible_ssh_common_args: >-
+          {% if login_is_private %}
+          -o ProxyCommand="gcloud compute start-iap-tunnel %h %p --listen-on-stdin --project={{ project }} --zone={{ zone }} --verbosity=warning"
+          {% endif %}
         groups: [remote_host]
-      when: (login_ip | default('')) | ansible.utils.ipaddr != false
+
+    - name: Add Controller node as host
+      ansible.builtin.add_host:
+        name: controller_node_host
+        ansible_host: "{{ controller_ip if not controller_is_private else controller_node_name }}"
+        ansible_ssh_common_args: >-
+          {% if controller_is_private %}
+          -o ProxyCommand="gcloud compute start-iap-tunnel %h %p --listen-on-stdin --project={{ project }} --zone={{ zone }} --verbosity=warning"
+          {% endif %}
+
 
     - name: Wait for host tasks
       ansible.builtin.include_tasks: tasks/wait-for-host.yml
       vars:
-        host_ip: "{{ login_ip }}"
+        host_to_wait: login_node_host
+        is_private: "{{ login_is_private }}"
         ansible_ssh_private_key_file: "/builder/home/.ssh/id_rsa"
 
     ## Cleanup and fail gracefully
@@ -297,7 +368,7 @@
       become: true
       changed_when: false
       failed_when: false
-      delegate_to: "{{ hostvars['localhost']['controller_ip']['stdout'] }}"
+      delegate_to: controller_node_host
       ansible.builtin.command: cat /slurm/scripts/setup.log
       register: setup_output
 
@@ -309,7 +380,7 @@
       become: true
       changed_when: false
       failed_when: false
-      delegate_to: "{{ hostvars['localhost']['controller_ip']['stdout'] }}"
+      delegate_to: controller_node_host
       ansible.builtin.command: cat /var/log/slurm/resume.log
       register: resume_output
 
@@ -321,7 +392,7 @@
       become: true
       changed_when: false
       failed_when: false
-      delegate_to: "{{ hostvars['localhost']['controller_ip']['stdout'] }}"
+      delegate_to: controller_node_host
       ansible.builtin.command: cat /var/log/slurm/suspend.log
       register: suspend_output
 
@@ -333,7 +404,7 @@
       become: true
       changed_when: false
       failed_when: false
-      delegate_to: "{{ hostvars['localhost']['controller_ip']['stdout'] }}"
+      delegate_to: controller_node_host
       ansible.builtin.command: cat /var/log/slurm/slurmctld.log
       register: slurmctld_output
 
@@ -345,7 +416,7 @@
       become: true
       changed_when: false
       failed_when: false
-      delegate_to: "{{ hostvars['localhost']['controller_ip']['stdout'] }}"
+      delegate_to: controller_node_host
       ansible.builtin.command: cat /var/log/slurm/slurmsync.log
       register: slurmsync_output
 

tools/cloud-build/daily-tests/ansible_playbooks/tasks/wait-for-host.yml

@@ -13,29 +13,35 @@
 # limitations under the License.
 ---
 
+- name: Set backward compatibility facts
+  ansible.builtin.set_fact:
+    _host_to_wait: "{{ host_to_wait | default(host_ip | default('')) }}"
+    _is_private: "{{ is_private | default(false) | bool }}"
+
 - name: Assert variables are defined
   ansible.builtin.assert:
     that:
-    - host_ip is defined
+    - _host_to_wait != ""
 
 - name: Wait for firewall to allow port 22 connection
   ansible.builtin.wait_for:
-    host: "{{ host_ip }}"
+    host: "{{ hostvars[_host_to_wait]['ansible_host'] | default(_host_to_wait) if _host_to_wait in hostvars else _host_to_wait }}"
     port: 22
     delay: 60
     timeout: 300
   delegate_to: localhost
   ignore_errors: true
   register: port_out
+  when: not _is_private
 
 - name: Check connection to remote host
   ansible.builtin.wait_for_connection:
     delay: 10
-  delegate_to: "{{ host_ip }}"
+  delegate_to: "{{ _host_to_wait }}"
   ignore_unreachable: true
   register: connect_out
 
 - name: Fail on bad connections
   ansible.builtin.fail:
-    msg: "Failed to connect to remote host {{ host_ip }}"
-  when: port_out is failed or connect_out is failed
+    msg: "Failed to connect to remote host {{ _host_to_wait }}"
+  when: (not _is_private and port_out is failed) or connect_out is failed