diff --git a/cmd/root.go b/cmd/root.go index a9e3754d78..73184740ad 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -53,7 +53,7 @@ HPC deployments on the Google Cloud Platform.`, logging.Fatal("cmd.Help function failed: %s", err) } }, - Version: "v1.59.1", + Version: "v1.60.0", Annotations: annotation, } ) diff --git a/community/modules/compute/htcondor-execute-point/versions.tf b/community/modules/compute/htcondor-execute-point/versions.tf index 91ab172585..986db448f5 100644 --- a/community/modules/compute/htcondor-execute-point/versions.tf +++ b/community/modules/compute/htcondor-execute-point/versions.tf @@ -29,6 +29,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.60.0" } } diff --git a/community/modules/compute/mig/versions.tf b/community/modules/compute/mig/versions.tf index fa039423d4..2092ec27af 100644 --- a/community/modules/compute/mig/versions.tf +++ b/community/modules/compute/mig/versions.tf @@ -22,6 +22,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:mig/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:mig/v1.60.0" } } diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf index 92612dfeee..d23296fc3c 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf @@ -24,6 +24,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-dynamic/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-dynamic/v1.60.0" } } diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf index cbcb844bbb..90679f507c 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 1.3" provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.60.0" } } diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf index 82cac0a4e4..40f8408bb8 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf @@ -24,6 +24,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.60.0" } } diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf index 9e97e7d8d5..e6896c90fd 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 1.3" provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.60.0" } } diff --git a/community/modules/database/slurm-cloudsql-federation/versions.tf b/community/modules/database/slurm-cloudsql-federation/versions.tf index 4b254c2dc9..deb54ee807 100644 --- a/community/modules/database/slurm-cloudsql-federation/versions.tf +++ b/community/modules/database/slurm-cloudsql-federation/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.60.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.60.0" } required_version = ">= 0.13.0" diff --git a/community/modules/file-system/cloud-storage-bucket/versions.tf b/community/modules/file-system/cloud-storage-bucket/versions.tf index b12798958f..f63eda0a10 100644 --- a/community/modules/file-system/cloud-storage-bucket/versions.tf +++ b/community/modules/file-system/cloud-storage-bucket/versions.tf @@ -30,10 +30,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.60.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.60.0" } required_version = ">= 0.14.0" } diff --git a/community/modules/file-system/nfs-server/versions.tf b/community/modules/file-system/nfs-server/versions.tf index 17d8d0eab6..a5e4ba06cf 100644 --- a/community/modules/file-system/nfs-server/versions.tf +++ b/community/modules/file-system/nfs-server/versions.tf @@ -30,7 +30,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.60.0" } required_version = ">= 0.14.0" diff --git a/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/community/modules/files/fsi-montecarlo-on-batch/versions.tf index 3ff40cfe03..0d5f0d989a 100644 --- a/community/modules/files/fsi-montecarlo-on-batch/versions.tf +++ b/community/modules/files/fsi-montecarlo-on-batch/versions.tf @@ -35,9 +35,9 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.60.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.60.0" } } diff --git a/community/modules/internal/slurm-gcp/login/versions.tf b/community/modules/internal/slurm-gcp/login/versions.tf index 72f33376b0..b6dd9a8cc5 100644 --- a/community/modules/internal/slurm-gcp/login/versions.tf +++ b/community/modules/internal/slurm-gcp/login/versions.tf @@ -24,6 +24,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-controller/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-controller/v1.60.0" } } diff --git a/community/modules/network/private-service-access/versions.tf b/community/modules/network/private-service-access/versions.tf index d0ab11cb5f..74c8f76e8b 100644 --- a/community/modules/network/private-service-access/versions.tf +++ b/community/modules/network/private-service-access/versions.tf @@ -30,11 +30,11 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.60.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.60.0" } required_version = ">= 1.2" diff --git a/community/modules/project/service-enablement/versions.tf b/community/modules/project/service-enablement/versions.tf index 1f01c7f99b..45d6e4b6ea 100644 --- a/community/modules/project/service-enablement/versions.tf +++ b/community/modules/project/service-enablement/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.60.0" } required_version = ">= 0.14.0" diff --git a/community/modules/pubsub/bigquery-sub/versions.tf b/community/modules/pubsub/bigquery-sub/versions.tf index 0649195dd8..ddb551688e 100644 --- a/community/modules/pubsub/bigquery-sub/versions.tf +++ b/community/modules/pubsub/bigquery-sub/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.60.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.60.0" } required_version = ">= 1.0" } diff --git a/community/modules/pubsub/topic/versions.tf b/community/modules/pubsub/topic/versions.tf index 7ce3040b82..a787882c88 100644 --- a/community/modules/pubsub/topic/versions.tf +++ b/community/modules/pubsub/topic/versions.tf @@ -27,6 +27,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:topic/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:topic/v1.60.0" } } diff --git a/community/modules/scheduler/htcondor-access-point/versions.tf b/community/modules/scheduler/htcondor-access-point/versions.tf index dea9db2b18..f23646afe4 100644 --- a/community/modules/scheduler/htcondor-access-point/versions.tf +++ b/community/modules/scheduler/htcondor-access-point/versions.tf @@ -30,7 +30,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.60.0" } required_version = ">= 1.1" diff --git a/community/modules/scheduler/htcondor-central-manager/versions.tf b/community/modules/scheduler/htcondor-central-manager/versions.tf index 3c51d5ec41..701ca37250 100644 --- a/community/modules/scheduler/htcondor-central-manager/versions.tf +++ b/community/modules/scheduler/htcondor-central-manager/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.60.0" } required_version = ">= 1.1.0" diff --git a/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/community/modules/scheduler/htcondor-pool-secrets/versions.tf index 016cba9e8f..ba6f53b770 100644 --- a/community/modules/scheduler/htcondor-pool-secrets/versions.tf +++ b/community/modules/scheduler/htcondor-pool-secrets/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.60.0" } required_version = ">= 1.3.0" diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf index 45022aa078..73d1beadb0 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-controller/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-controller/v1.60.0" } } diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf index 205cef32ab..d4c11d11b6 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf @@ -24,6 +24,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-login/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-login/v1.60.0" } } diff --git a/community/modules/scripts/wait-for-startup/versions.tf b/community/modules/scripts/wait-for-startup/versions.tf index c6717c155f..de98b93e57 100644 --- a/community/modules/scripts/wait-for-startup/versions.tf +++ b/community/modules/scripts/wait-for-startup/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.60.0" } required_version = ">= 0.14.0" diff --git a/community/modules/scripts/windows-startup-script/versions.tf b/community/modules/scripts/windows-startup-script/versions.tf index 81f272e9ef..44e50cc859 100644 --- a/community/modules/scripts/windows-startup-script/versions.tf +++ b/community/modules/scripts/windows-startup-script/versions.tf @@ -16,7 +16,7 @@ terraform { provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.60.0" } required_version = ">= 0.14.0" diff --git a/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml b/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml index dc97c17d06..6bb2a3c1b6 100644 --- a/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml +++ b/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-blueprint.yaml @@ -84,6 +84,36 @@ deployment_groups: Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1 Pin: version 1.17.7-1 Pin-Priority: 100 + + # The following holds NVIDIA software that was already installed on the + # accelerator base image to be the same driver version. This reduces the + # risk of a driver version mismatch. + # Additional packages are held by: + # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml + - type: ansible-local + destination: hold-nvidia-packages.yml + content: | + --- + - name: Hold nvidia packages + hosts: all + become: true + vars: + nvidia_packages_to_hold: + - libnvidia-cfg1-*-server + - libnvidia-compute-*-server + - libnvidia-nscq-* + - nvidia-compute-utils-*-server + - nvidia-fabricmanager-* + - nvidia-utils-*-server + tasks: + - name: Hold nvidia packages + ansible.builtin.command: + argv: + - apt-mark + - hold + - "{{ item }}" + loop: "{{ nvidia_packages_to_hold }}" + # it is important that kernel upgrades do not occur before running the # solution for building Slurm (which doesn't handle them well on the fly) # if you follow this rule, any module which supports DKMS will be @@ -181,7 +211,7 @@ deployment_groups: destination: configure_gpu_monitoring.yml content: | --- - - name: Install NVIDIA DCGM and Configure Ops Agent + - name: Install CUDA & DCGM & Configure Ops Agent hosts: all become: true vars: @@ -193,9 +223,6 @@ deployment_groups: nvidia_packages: - cuda-toolkit-12-8 - datacenter-gpu-manager-4-cuda12 - - libnvidia-cfg1-570-server - - libnvidia-nscq-570 - - nvidia-compute-utils-570-server tasks: - name: Download NVIDIA repository package ansible.builtin.get_url: @@ -223,16 +250,11 @@ deployment_groups: Package: * Pin: release l=NVIDIA CUDA Pin-Priority: 400 - - name: Install NVIDIA fabric and CUDA + - name: Install CUDA & DCGM ansible.builtin.apt: name: "{{ item }}" update_cache: true loop: "{{ nvidia_packages }}" - - name: Freeze NVIDIA fabric and CUDA - ansible.builtin.dpkg_selections: - name: "{{ item }}" - selection: hold - loop: "{{ nvidia_packages }}" - name: Create nvidia-persistenced override directory ansible.builtin.file: path: /etc/systemd/system/nvidia-persistenced.service.d @@ -271,6 +293,7 @@ deployment_groups: name: nvidia-persistenced.service state: stopped enabled: false + - type: ansible-local destination: install_dmabuf.yml content: | diff --git a/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml b/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml index 7f4484977d..0cd00e1925 100644 --- a/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml +++ b/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml @@ -69,6 +69,36 @@ deployment_groups: Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1 Pin: version 1.17.7-1 Pin-Priority: 100 + + # The following holds NVIDIA software that was already installed on the + # accelerator base image to be the same driver version. This reduces the + # risk of a driver version mismatch. + # Additional packages are held by: + # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml + - type: ansible-local + destination: hold-nvidia-packages.yml + content: | + --- + - name: Hold nvidia packages + hosts: all + become: true + vars: + nvidia_packages_to_hold: + - libnvidia-cfg1-*-server + - libnvidia-compute-*-server + - libnvidia-nscq-* + - nvidia-compute-utils-*-server + - nvidia-fabricmanager-* + - nvidia-utils-*-server + tasks: + - name: Hold nvidia packages + ansible.builtin.command: + argv: + - apt-mark + - hold + - "{{ item }}" + loop: "{{ nvidia_packages_to_hold }}" + - type: data destination: /var/tmp/slurm_vars.json content: | @@ -93,7 +123,7 @@ deployment_groups: -i localhost, --limit localhost --connection=local \ -e @/var/tmp/slurm_vars.json \ ansible/playbook.yml - # this duplicates the ulimits configuration of the HPC VM Image + # this duplicates the ulimits configuration of the HPC VM Image - type: data destination: /etc/security/limits.d/99-unlimited.conf content: | @@ -103,11 +133,12 @@ deployment_groups: * - nofile 1048576 * - cpu unlimited * - rtprio unlimited + - type: ansible-local - destination: configure_gpu.yml + destination: install_cuda_dcgm.yml content: | --- - - name: Install NVIDIA packages + - name: Install CUDA & DCGM hosts: all become: true vars: @@ -118,9 +149,6 @@ deployment_groups: nvidia_packages: - cuda-toolkit-12-8 - datacenter-gpu-manager-4-cuda12 - - libnvidia-cfg1-570-server - - libnvidia-nscq-570 - - nvidia-compute-utils-570-server tasks: - name: Download NVIDIA repository package ansible.builtin.get_url: @@ -130,7 +158,6 @@ deployment_groups: ansible.builtin.apt: deb: "{{ cuda_repo_filename }}" state: present - - name: Reduce NVIDIA repository priority ansible.builtin.copy: dest: /etc/apt/preferences.d/cuda-repository-pin-600 @@ -149,16 +176,11 @@ deployment_groups: Package: * Pin: release l=NVIDIA CUDA Pin-Priority: 400 - - name: Install NVIDIA fabric and CUDA + - name: Install CUDA & DCGM ansible.builtin.apt: name: "{{ item }}" update_cache: true loop: "{{ nvidia_packages }}" - - name: Freeze NVIDIA fabric and CUDA - ansible.builtin.dpkg_selections: - name: "{{ item }}" - selection: hold - loop: "{{ nvidia_packages }}" - name: Create nvidia-persistenced override directory ansible.builtin.file: path: /etc/systemd/system/nvidia-persistenced.service.d diff --git a/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml b/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml index f319dcaf20..dd51f2a56a 100644 --- a/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml +++ b/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml @@ -70,6 +70,36 @@ deployment_groups: Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1 Pin: version 1.17.7-1 Pin-Priority: 100 + + # The following holds NVIDIA software that was already installed on the + # accelerator base image to be the same driver version. This reduces the + # risk of a driver version mismatch. + # Additional packages are held by: + # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml + - type: ansible-local + destination: hold-nvidia-packages.yml + content: | + --- + - name: Hold nvidia packages + hosts: all + become: true + vars: + nvidia_packages_to_hold: + - libnvidia-cfg1-*-server + - libnvidia-compute-*-server + - libnvidia-nscq-* + - nvidia-compute-utils-*-server + - nvidia-fabricmanager-* + - nvidia-utils-*-server + tasks: + - name: Hold nvidia packages + ansible.builtin.command: + argv: + - apt-mark + - hold + - "{{ item }}" + loop: "{{ nvidia_packages_to_hold }}" + - type: data destination: /var/tmp/slurm_vars.json content: | @@ -104,11 +134,12 @@ deployment_groups: * - nofile 1048576 * - cpu unlimited * - rtprio unlimited + - type: ansible-local - destination: configure_gpu.yml + destination: install_cuda_dcgm.yml content: | --- - - name: Install NVIDIA packages + - name: Install CUDA & DCGM hosts: all become: true vars: @@ -119,9 +150,6 @@ deployment_groups: nvidia_packages: - cuda-toolkit-12-8 - datacenter-gpu-manager-4-cuda12 - - libnvidia-cfg1-570-server - - libnvidia-nscq-570 - - nvidia-compute-utils-570-server tasks: - name: Download NVIDIA repository package ansible.builtin.get_url: @@ -131,7 +159,6 @@ deployment_groups: ansible.builtin.apt: deb: "{{ cuda_repo_filename }}" state: present - - name: Reduce NVIDIA repository priority ansible.builtin.copy: dest: /etc/apt/preferences.d/cuda-repository-pin-600 @@ -150,16 +177,11 @@ deployment_groups: Package: * Pin: release l=NVIDIA CUDA Pin-Priority: 400 - - name: Install NVIDIA fabric and CUDA + - name: Install CUDA & DCGM ansible.builtin.apt: name: "{{ item }}" update_cache: true loop: "{{ nvidia_packages }}" - - name: Freeze NVIDIA fabric and CUDA - ansible.builtin.dpkg_selections: - name: "{{ item }}" - selection: hold - loop: "{{ nvidia_packages }}" - name: Create nvidia-persistenced override directory ansible.builtin.file: path: /etc/systemd/system/nvidia-persistenced.service.d @@ -225,7 +247,6 @@ deployment_groups: source_image_project_id: [$(vars.base_image.project)] image_family: $(vars.instance_image.family) omit_external_ip: false - # Unattended upgrades are disabled in this blueprint so that software does not # get updated daily and lead to potential instability in the cluster environment. # diff --git a/examples/machine-learning/build-service-images/a3m/blueprint.yaml b/examples/machine-learning/build-service-images/a3m/blueprint.yaml index 858384b042..226a945453 100644 --- a/examples/machine-learning/build-service-images/a3m/blueprint.yaml +++ b/examples/machine-learning/build-service-images/a3m/blueprint.yaml @@ -37,6 +37,36 @@ deployment_groups: Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1 Pin: version 1.17.7-1 Pin-Priority: 100 + + # The following holds NVIDIA software that was already installed on the + # accelerator base image to be the same driver version. This reduces the + # risk of a driver version mismatch. + # Additional packages are held by: + # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml + - type: ansible-local + destination: hold-nvidia-packages.yml + content: | + --- + - name: Hold nvidia packages + hosts: all + become: true + vars: + nvidia_packages_to_hold: + - libnvidia-cfg1-*-server + - libnvidia-compute-*-server + - libnvidia-nscq-* + - nvidia-compute-utils-*-server + - nvidia-fabricmanager-* + - nvidia-utils-*-server + tasks: + - name: Hold nvidia packages + ansible.builtin.command: + argv: + - apt-mark + - hold + - "{{ item }}" + loop: "{{ nvidia_packages_to_hold }}" + - type: shell destination: hold_google_services.sh content: | @@ -118,7 +148,7 @@ deployment_groups: destination: configure_gpu_monitoring.yml content: | --- - - name: Install NVIDIA DCGM and Configure Ops Agent + - name: Install CUDA & DCGM & Configure Ops Agent hosts: all become: true vars: @@ -130,9 +160,6 @@ deployment_groups: nvidia_packages: - cuda-toolkit-12-8 - datacenter-gpu-manager-4-cuda12 - - libnvidia-cfg1-570-server - - libnvidia-nscq-570 - - nvidia-compute-utils-570-server tasks: - name: Download NVIDIA repository package ansible.builtin.get_url: @@ -160,16 +187,11 @@ deployment_groups: Package: * Pin: release l=NVIDIA CUDA Pin-Priority: 400 - - name: Install NVIDIA fabric and CUDA + - name: Install CUDA & DCGM ansible.builtin.apt: name: "{{ item }}" update_cache: true loop: "{{ nvidia_packages }}" - - name: Freeze NVIDIA fabric and CUDA - ansible.builtin.dpkg_selections: - name: "{{ item }}" - selection: hold - loop: "{{ nvidia_packages }}" - name: Create nvidia-persistenced override directory ansible.builtin.file: path: /etc/systemd/system/nvidia-persistenced.service.d @@ -208,6 +230,7 @@ deployment_groups: name: nvidia-persistenced.service state: stopped enabled: false + - type: ansible-local destination: install_dmabuf.yml content: | @@ -300,7 +323,7 @@ deployment_groups: disk_size: 100 machine_type: c2-standard-8 - source_image: ubuntu-accelerator-2204-amd64-with-nvidia-570-v20250712 + source_image: ubuntu-accelerator-2204-amd64-with-nvidia-570-v20250722 source_image_project_id: [ubuntu-os-accelerator-images] image_family: $(vars.family) diff --git a/examples/machine-learning/build-service-images/common/blueprint.yaml b/examples/machine-learning/build-service-images/common/blueprint.yaml index acc7260bec..eced20915c 100644 --- a/examples/machine-learning/build-service-images/common/blueprint.yaml +++ b/examples/machine-learning/build-service-images/common/blueprint.yaml @@ -37,6 +37,36 @@ deployment_groups: Package: nvidia-container-toolkit nvidia-container-toolkit-base libnvidia-container-tools libnvidia-container1 Pin: version 1.17.7-1 Pin-Priority: 100 + + # The following holds NVIDIA software that was already installed on the + # accelerator base image to be the same driver version. This reduces the + # risk of a driver version mismatch. + # Additional packages are held by: + # https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/ansible/group_vars/os_ubuntu.yml + - type: ansible-local + destination: hold-nvidia-packages.yml + content: | + --- + - name: Hold nvidia packages + hosts: all + become: true + vars: + nvidia_packages_to_hold: + - libnvidia-cfg1-*-server + - libnvidia-compute-*-server + - libnvidia-nscq-* + - nvidia-compute-utils-*-server + - nvidia-fabricmanager-* + - nvidia-utils-*-server + tasks: + - name: Hold nvidia packages + ansible.builtin.command: + argv: + - apt-mark + - hold + - "{{ item }}" + loop: "{{ nvidia_packages_to_hold }}" + - type: data destination: /var/tmp/slurm_vars.json content: | @@ -55,10 +85,10 @@ deployment_groups: - $(vars.runner_install_slurm) - $(vars.runner_setup_hpc_vm_image_ulimits) - type: ansible-local - destination: configure_gpu.yml + destination: install_cuda_dcgm.yml content: | --- - - name: Install NVIDIA packages + - name: Install CUDA & DCGM hosts: all become: true vars: @@ -68,10 +98,7 @@ deployment_groups: enable_nvidia_dcgm: false nvidia_packages: - cuda-toolkit-12-8 - - datacenter-gpu-manager - - libnvidia-cfg1-570-server - - libnvidia-nscq-570 - - nvidia-compute-utils-570-server + - datacenter-gpu-manager-4-cuda12 tasks: - name: Download NVIDIA repository package ansible.builtin.get_url: @@ -99,16 +126,11 @@ deployment_groups: Package: * Pin: release l=NVIDIA CUDA Pin-Priority: 400 - - name: Install NVIDIA fabric and CUDA + - name: Install CUDA & DCGM ansible.builtin.apt: name: "{{ item }}" update_cache: true loop: "{{ nvidia_packages }}" - - name: Freeze NVIDIA fabric and CUDA - ansible.builtin.dpkg_selections: - name: "{{ item }}" - selection: hold - loop: "{{ nvidia_packages }}" - name: Create nvidia-persistenced override directory ansible.builtin.file: path: /etc/systemd/system/nvidia-persistenced.service.d diff --git a/modules/compute/gke-node-pool/versions.tf b/modules/compute/gke-node-pool/versions.tf index 7f995ef07f..0755e65311 100644 --- a/modules/compute/gke-node-pool/versions.tf +++ b/modules/compute/gke-node-pool/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.60.0" } provider_meta "google-beta" { module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.45.0" diff --git a/modules/compute/vm-instance/versions.tf b/modules/compute/vm-instance/versions.tf index cba85144df..dd892d9665 100644 --- a/modules/compute/vm-instance/versions.tf +++ b/modules/compute/vm-instance/versions.tf @@ -31,10 +31,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.60.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.60.0" } required_version = ">= 1.3.0" diff --git a/modules/file-system/filestore/versions.tf b/modules/file-system/filestore/versions.tf index b89b577221..0f22940d3e 100644 --- a/modules/file-system/filestore/versions.tf +++ b/modules/file-system/filestore/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.60.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.60.0" } required_version = ">= 1.3.0" diff --git a/modules/file-system/gke-persistent-volume/versions.tf b/modules/file-system/gke-persistent-volume/versions.tf index ce65112b9b..dfe6d7bfa3 100644 --- a/modules/file-system/gke-persistent-volume/versions.tf +++ b/modules/file-system/gke-persistent-volume/versions.tf @@ -29,6 +29,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.60.0" } } diff --git a/modules/file-system/gke-storage/versions.tf b/modules/file-system/gke-storage/versions.tf index f9f784815c..b2bfd2c6cb 100644 --- a/modules/file-system/gke-storage/versions.tf +++ b/modules/file-system/gke-storage/versions.tf @@ -16,6 +16,6 @@ terraform { required_version = ">= 1.5" provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.60.0" } } diff --git a/modules/file-system/managed-lustre/versions.tf b/modules/file-system/managed-lustre/versions.tf index 8c008f4102..448b7ed913 100644 --- a/modules/file-system/managed-lustre/versions.tf +++ b/modules/file-system/managed-lustre/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.60.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.60.0" } required_version = ">= 1.3.0" diff --git a/modules/monitoring/dashboard/versions.tf b/modules/monitoring/dashboard/versions.tf index 6fd01ef3cf..4e08257f7e 100644 --- a/modules/monitoring/dashboard/versions.tf +++ b/modules/monitoring/dashboard/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.60.0" } required_version = ">= 0.14.0" diff --git a/modules/network/firewall-rules/versions.tf b/modules/network/firewall-rules/versions.tf index 4c5413c061..1cba8afb9a 100644 --- a/modules/network/firewall-rules/versions.tf +++ b/modules/network/firewall-rules/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.60.0" } required_version = ">= 1.5" diff --git a/modules/network/pre-existing-subnetwork/versions.tf b/modules/network/pre-existing-subnetwork/versions.tf index 1952e10f83..a06989cd0c 100644 --- a/modules/network/pre-existing-subnetwork/versions.tf +++ b/modules/network/pre-existing-subnetwork/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.60.0" } required_version = ">= 1.5" diff --git a/modules/network/pre-existing-vpc/versions.tf b/modules/network/pre-existing-vpc/versions.tf index fd7d087a1f..034dcd9952 100644 --- a/modules/network/pre-existing-vpc/versions.tf +++ b/modules/network/pre-existing-vpc/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.60.0" } required_version = ">= 1.5" diff --git a/modules/scheduler/batch-login-node/versions.tf b/modules/scheduler/batch-login-node/versions.tf index 1d65d8bae4..93ee2f738a 100644 --- a/modules/scheduler/batch-login-node/versions.tf +++ b/modules/scheduler/batch-login-node/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.60.0" } required_version = ">= 0.14.0" diff --git a/modules/scheduler/gke-cluster/versions.tf b/modules/scheduler/gke-cluster/versions.tf index 7996a7f4cc..b7f1fd5a84 100644 --- a/modules/scheduler/gke-cluster/versions.tf +++ b/modules/scheduler/gke-cluster/versions.tf @@ -30,7 +30,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-cluster/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:gke-cluster/v1.60.0" } provider_meta "google-beta" { diff --git a/modules/scheduler/pre-existing-gke-cluster/versions.tf b/modules/scheduler/pre-existing-gke-cluster/versions.tf index 947debf316..567757d4f9 100644 --- a/modules/scheduler/pre-existing-gke-cluster/versions.tf +++ b/modules/scheduler/pre-existing-gke-cluster/versions.tf @@ -23,7 +23,7 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.60.0" } required_version = ">= 1.3" diff --git a/modules/scripts/startup-script/versions.tf b/modules/scripts/startup-script/versions.tf index 1e359535e0..f8878fe452 100644 --- a/modules/scripts/startup-script/versions.tf +++ b/modules/scripts/startup-script/versions.tf @@ -30,7 +30,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:startup-script/v1.59.1" + module_name = "blueprints/terraform/hpc-toolkit:startup-script/v1.60.0" } required_version = ">= 1.5"