Pin DCGM to version 4.5.0 (GoogleCloudPlatform#5124)

Author: LAVEEN

examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml

@@ -184,8 +184,12 @@ deployment_groups:
           apt update -y
           apt install -y cuda-toolkit-12-8
           apt install -y nvidia-container-toolkit
-          apt install -y datacenter-gpu-manager-4-cuda12
-          apt install -y datacenter-gpu-manager-4-dev
+          #Pin the package to latest DCGMI version 4.5.0
+          apt install -y datacenter-gpu-manager-4-cuda12=1:4.5.0-1
+          apt install -y datacenter-gpu-manager-4-dev=1:4.5.0-1
+
+          #Hold packages to prevent auto-updates
+          apt-mark hold datacenter-gpu-manager-4-cuda12 datacenter-gpu-manager-4-dev
       # this duplicates the ulimits configuration of the HPC VM Image
       - type: data
         destination: /etc/security/limits.d/99-unlimited.conf

examples/machine-learning/a3-megagpu-8g/a3mega-slurm-gcsfuse-lssd-blueprint.yaml

@@ -479,8 +479,9 @@ deployment_groups:
               enable_nvidia_dcgm: $(vars.enable_nvidia_dcgm)
               nvidia_packages:
               - cuda-toolkit-12-8
-              - datacenter-gpu-manager-4-cuda12
-              - datacenter-gpu-manager-4-dev
+              #Pin the package to latest DCGMI version 4.5.0
+              - datacenter-gpu-manager-4-cuda12=1:4.5.0-1
+              - datacenter-gpu-manager-4-dev=1:4.5.0-1
               - libnvidia-cfg1-570-server
               - libnvidia-nscq-570
               - nvidia-compute-utils-570-server

examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml

@@ -159,8 +159,11 @@ deployment_groups:
           set -e -o pipefail
           add-nvidia-repositories -y
           apt install -y cuda-toolkit-12-8
-          apt install -y datacenter-gpu-manager-4-cuda12
-          apt install -y datacenter-gpu-manager-4-dev
+          #Pin the package to latest DCGMI version 4.5.0
+          apt install -y datacenter-gpu-manager-4-cuda12=1:4.5.0-1
+          apt install -y datacenter-gpu-manager-4-dev=1:4.5.0-1
+          #Hold packages to prevent auto-updates
+          apt-mark hold datacenter-gpu-manager-4-cuda12 datacenter-gpu-manager-4-dev
       - type: ansible-local
         destination: settings_nvidia_dcgm.yml
         content: |

examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml

@@ -161,8 +161,12 @@ deployment_groups:
           add-nvidia-repositories -y
           apt update -y
           apt install -y cuda-toolkit-12-8
-          apt install -y datacenter-gpu-manager-4-cuda12
-          apt install -y datacenter-gpu-manager-4-dev
+          #Pin the package to latest DCGMI version 4.5.0
+          apt install -y datacenter-gpu-manager-4-cuda12=1:4.5.0-1
+          apt install -y datacenter-gpu-manager-4-dev=1:4.5.0-1
+
+          #Hold packages to prevent auto-updates
+          apt-mark hold datacenter-gpu-manager-4-cuda12 datacenter-gpu-manager-4-dev
       - type: ansible-local
         destination: configure_cuda_dcgm.yml
         content: |

examples/machine-learning/a4x-highgpu-4g/a4xhigh-slurm-blueprint.yaml

@@ -177,8 +177,9 @@ deployment_groups:
               cuda_repo_filename: /tmp/{{ cuda_repo_url | basename }}
               nvidia_packages:
               - cuda-toolkit-12-8
-              - datacenter-gpu-manager-4-cuda12
-              - datacenter-gpu-manager-4-dev
+              #Pin the package to latest DCGMI version 4.5.0
+              - datacenter-gpu-manager-4-cuda12=1:4.5.0-1
+              - datacenter-gpu-manager-4-dev=1:4.5.0-1
             tasks:
             - name: Download NVIDIA repository package
               ansible.builtin.get_url: