Merge pull request GoogleCloudPlatform#4578 from arpit974/add_network_wait_solution

Adding network-wait solution to Ubuntu based slurm clusters.
It waits until network-online.target is reached (at least 1 NIC is up and routable)
It sets itself up to run before google-startup-scripts.service
It runs systemd-networkd-wait-online on all the GPU NICs, enforcing routability of each but also giving up after 180 seconds

Author: arpit974

examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml

@@ -63,6 +63,7 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       install_ansible: true
+      enable_gpu_network_wait_online: true
       configure_ssh_host_patterns:
       - 10.0.0.*
       - 10.1.0.*

examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml

@@ -59,6 +59,7 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       install_ansible: true
+      enable_gpu_network_wait_online: true
       docker:
         enabled: true
         world_writable: true

examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml

@@ -60,6 +60,7 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       install_ansible: true
+      enable_gpu_network_wait_online: true
       docker:
         enabled: true
         world_writable: true

modules/scripts/startup-script/files/install_gpu_network_wait_online.yml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ---
-- name: Wait until all network interfaces are online
+- name: Wait until all network interfaces are online for A3/A4 variants
   hosts: all
   become: true
   tasks:
@@ -32,23 +32,104 @@
         Before=google-startup-scripts.service
 
         [Service]
-        ExecCondition=/bin/bash -c '/usr/bin/curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetata/v1/instance/machine-type | grep -q "/a3-highgpu-8g$"'
+        # Condition to only run on A3 High machines
+        ExecCondition=/bin/bash -c '/usr/bin/curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -q "/a3-highgpu-8g$"'
         ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -i enp0s12 -i enp6s0 -i enp12s0 -i enp134s0 -i enp140s0 -o routable --timeout=180
         ExecStartPost=/bin/sleep 30
 
         [Install]
         WantedBy=multi-user.target
     notify:
     - Reload SystemD
-    - Enable A3 High delay
+
+  - name: Create SystemD service for A3 Mega networking
+    when: ansible_os_family == "Debian"
+    ansible.builtin.copy:
+      dest: /etc/systemd/system/delay-a3-mega.service
+      owner: root
+      group: root
+      mode: "0644"
+      content: |
+        [Unit]
+        Description=Delay A3 Mega boot until all network interfaces are routable
+        After=network-online.target
+        Wants=network-online.target
+        Before=google-startup-scripts.service
+
+        [Service]
+        # Condition to only run on A3 Mega machines
+        ExecCondition=/bin/bash -c '/usr/bin/curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -q "/a3-megagpu-8g$"'
+        ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -i enp0s12 -i enp6s0f0 -i enp7s0f0 -i enp13s0f0 -i enp14s0f0 -i enp134s0f0 -i enp135s0f0 -i enp141s0f0 -i enp142s0f0 -o routable --timeout=180
+        ExecStartPost=/bin/sleep 30
+
+        [Install]
+        WantedBy=multi-user.target
+    notify:
+    - Reload SystemD
+
+  - name: Create SystemD service for A3 Ultra networking
+    when: ansible_os_family == "Debian"
+    ansible.builtin.copy:
+      dest: /etc/systemd/system/delay-a3-ultra.service
+      owner: root
+      group: root
+      mode: "0644"
+      content: |
+        [Unit]
+        Description=Delay A3 Ultra boot until all network interfaces are routable
+        After=network-online.target
+        Wants=network-online.target
+        Before=google-startup-scripts.service
+
+        [Service]
+        # Condition to only run on A3 Ultra machines
+        ExecCondition=/bin/bash -c '/usr/bin/curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -q "/a3-ultragpu-8g$"'
+        ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -i enp0s19 -i enp192s20 -i gpu0rdma0 -i gpu1rdma0 - gpu2rdma0 -i gpu3rdma0 -i gpu4rdma0 -i gpu5rdma0 -i gpu6rdma0 -i gpu7rdma0 -o routable --timeout=180
+        ExecStartPost=/bin/sleep 30
+
+        [Install]
+        WantedBy=multi-user.target
+    notify:
+    - Reload SystemD
+
+  - name: Create SystemD service for A4 High networking
+    when: ansible_os_family == "Debian"
+    ansible.builtin.copy:
+      dest: /etc/systemd/system/delay-a4-high.service
+      owner: root
+      group: root
+      mode: "0644"
+      content: |
+        [Unit]
+        Description=Delay A4 High boot until all network interfaces are routable
+        After=network-online.target
+        Wants=network-online.target
+        Before=google-startup-scripts.service
+
+        [Service]
+        # Condition to only run on A4 High machines
+        ExecCondition=/bin/bash -c '/usr/bin/curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -q "/a4-highgpu-8g$"'
+        ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -i enp0s19 -i enp192s20 -i gpu0rdma0 -i gpu1rdma0 - gpu2rdma0 -i gpu3rdma0 -i gpu4rdma0 -i gpu5rdma0 -i gpu6rdma0 -i gpu7rdma0 -o routable --timeout=180
+        ExecStartPost=/bin/sleep 30
+        # To debug interface names on an A4 instance, run: ip link show
+
+        [Install]
+        WantedBy=multi-user.target
+    notify:
+    - Reload SystemD
+
+  - name: Enable A3/A4 delay services
+    when: ansible_os_family == "Debian"
+    ansible.builtin.systemd_service:
+      name: "{{ item }}"
+      enabled: true
+    loop:
+    - delay-a3-high.service
+    - delay-a3-mega.service
+    - delay-a3-ultra.service
+    - delay-a4-high.service
+
   handlers:
   - name: Reload SystemD
     ansible.builtin.systemd:
       daemon_reload: true
-  post_tasks:
-  - name: Enable A3 High delay
-    # by the time this startup-script executes, we don't care if it's started
-    # just enabled
-    ansible.builtin.systemd_service:
-      name: delay-a3-high
-      enabled: true