Merge pull request GoogleCloudPlatform#4704 from samskillman/feat/h4d-irdma-health-check

Adds an irdma health check to h4d nodes

Author: samskillman

examples/hpc-slurm-h4d.yaml

@@ -128,10 +128,43 @@ deployment_groups:
       machine_type: n2-standard-4
       enable_login_public_ips: true
 
+  - id: controller_startup
+    source: modules/scripts/startup-script
+    settings:
+      runners:
+      - type: shell
+        destination: stage_scripts.sh
+        content: |
+          #!/bin/bash
+          SLURM_ROOT=/opt/apps/adm/slurm
+          PARTITION_NAME=$(h4d_partition.partitions[0].partition_name)
+          mkdir -m 0755 -p "${SLURM_ROOT}/scripts"
+
+          # enable the use of password-free sudo within Slurm jobs on all compute nodes
+          # feature is restricted to users with OS Admin Login IAM role
+          # https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin
+          mkdir -p "${SLURM_ROOT}/prolog_slurmd.d"
+          mkdir -p "${SLURM_ROOT}/epilog_slurmd.d"
+          curl -s -o "${SLURM_ROOT}/scripts/sudo-oslogin" \
+              https://raw.githubusercontent.com/GoogleCloudPlatform/slurm-gcp/master/tools/prologs-epilogs/sudo-oslogin
+          chmod 0755 "${SLURM_ROOT}/scripts/sudo-oslogin"
+
+          ln -s "${SLURM_ROOT}/scripts/sudo-oslogin" "${SLURM_ROOT}/prolog_slurmd.d/sudo-oslogin.prolog_slurmd"
+          ln -s "${SLURM_ROOT}/scripts/sudo-oslogin" "${SLURM_ROOT}/epilog_slurmd.d/sudo-oslogin.epilog_slurmd"
+
+          # enable an RDMA health check that runs prior to any H4D job
+          mkdir -p "${SLURM_ROOT}/partition-${PARTITION_NAME}-prolog_slurmd.d"
+          curl -s -o "${SLURM_ROOT}/partition-${PARTITION_NAME}-prolog_slurmd.d/irdma_health_check.prolog_slurmd" \
+              https://raw.githubusercontent.com/GoogleCloudPlatform/slurm-gcp/master/tools/prologs-epilogs/irdma_health_check
+          chmod 0755 "${SLURM_ROOT}/partition-${PARTITION_NAME}-prolog_slurmd.d/irdma_health_check.prolog_slurmd"
+
+
   - id: slurm_controller
     source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller
     use: [h4d-slurm-net-0, h4d_partition, slurm_login, homefs]
     settings:
       enable_controller_public_ips: true
+      controller_startup_script: $(controller_startup.startup_script)
+      enable_external_prolog_epilog: true
       cloud_parameters:
         unkillable_step_timeout: 900