Merge pull request GoogleCloudPlatform#4275 from abbas1902/qf

Revamp install_cloud_rdma_drivers startup script

Author: abbas1902

examples/h4d-vm.yaml

@@ -69,7 +69,6 @@ deployment_groups:
     settings:
       configure_ssh_host_patterns:
       - $(vars.hostname_prefix)-*
-      install_cloud_rdma_drivers: true
       set_ofi_cloud_rdma_tunables: true
       local_ssd_filesystem:
         fs_type: ext4

examples/hpc-slurm-h4d.yaml

@@ -69,7 +69,6 @@ deployment_groups:
   - id: h4d_startup
     source: modules/scripts/startup-script
     settings:
-      install_cloud_rdma_drivers: true
       set_ofi_cloud_rdma_tunables: true
       local_ssd_filesystem:
         fs_type: ext4

modules/scripts/startup-script/README.md

@@ -326,7 +326,7 @@ No modules.
 | <a name="input_http_proxy"></a> [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no |
 | <a name="input_install_ansible"></a> [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no |
 | <a name="input_install_cloud_ops_agent"></a> [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no |
-| <a name="input_install_cloud_rdma_drivers"></a> [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. | `bool` | `false` | no |
+| <a name="input_install_cloud_rdma_drivers"></a> [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no |
 | <a name="input_install_docker"></a> [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no |
 | <a name="input_install_stackdriver_agent"></a> [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no |
 | <a name="input_labels"></a> [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes |

modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh

@@ -17,32 +17,21 @@ set -e -o pipefail
 OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')"
 OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')"
 OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')"
+REBOOT_FILE="/etc/.rdma_reboot"
 
 if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then
 	KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')"
 
 	# For images that do not already have Cloud RDMA drivers installed
-	if [ -z "${KMOD_VERSION}" ]; then
+	if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then
+		sudo dnf update -y
 		sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y
 		sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y
-		sudo rmmod idpf
-		sudo rmmod irdma
-		sudo modprobe idpf
-		exit 0
-	else
-		# Downloading the RDMA packages and upgrading existing drivers
-		sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y
-		sudo dnf upgrade kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y
-		NEW_KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')"
-
-		# Restarting drivers so they can use RDMA without needing a reboot (specifically checking for kmod package updates)
-		if [ "${KMOD_VERSION}" != "${NEW_KMOD_VERSION}" ]; then
-			sudo rmmod idpf
-			sudo rmmod irdma
-			sudo modprobe idpf
-		fi
-		exit 0
+		sudo touch "${REBOOT_FILE}"
+		reboot
 	fi
+	echo "This image has IRDMA packages already installed, exiting."
+	exit 0
 else
 	echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8."
 	exit 1

modules/scripts/startup-script/variables.tf

@@ -280,7 +280,7 @@ variable "http_no_proxy" {
 }
 
 variable "install_cloud_rdma_drivers" {
-  description = "If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8."
+  description = "If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image."
   type        = bool
   default     = false
 }