diff --git a/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml b/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml index cdb49de95f..f5631e8fad 100644 --- a/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml +++ b/examples/machine-learning/a3-ultragpu-8g/a3ultra-slurm-blueprint.yaml @@ -24,7 +24,7 @@ vars: # Image settings base_image: project: ubuntu-os-accelerator-images - family: ubuntu-accelerator-2204-amd64-with-nvidia-570 + image: ubuntu-accelerator-2204-amd64-with-nvidia-570-v20250712 image_build_machine_type: n2-standard-16 build_slurm_from_git_ref: 6.10.0 # Cluster env settings @@ -147,6 +147,24 @@ deployment_groups: ansible.builtin.apt: deb: "{{ cuda_repo_filename }}" state: present + # The following 2 tasks work around a temporary issue with Ubuntu + # packaging of NVIDIA 570 driver series for kernel 6.8.0-1032 + # This command ensures that any holds are removed before attempting an upgrade. + # We ignore failures in case the packages were not held. + - name: Unhold NVIDIA driver packages + ansible.builtin.command: + cmd: apt-mark unhold linux-modules-nvidia-570-server-open-gcp linux-modules-nvidia-570-server-open-6.8.0-1032-gcp + become: true + changed_when: false + failed_when: false + - name: Install latest NVIDIA driver metapackage and kernel module + ansible.builtin.apt: + name: + - linux-modules-nvidia-570-server-open-gcp + - linux-modules-nvidia-570-server-open-6.8.0-1032-gcp + state: latest + update_cache: yes + become: true - name: Reduce NVIDIA repository priority ansible.builtin.copy: dest: /etc/apt/preferences.d/cuda-repository-pin-600 @@ -236,7 +254,7 @@ deployment_groups: settings: disk_size: $(vars.disk_size_gb) machine_type: $(vars.image_build_machine_type) - source_image_family: $(vars.base_image.family) + source_image: $(vars.base_image.image) source_image_project_id: [$(vars.base_image.project)] image_family: $(vars.instance_image.family) omit_external_ip: false diff --git a/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml b/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml index dd456d9648..ebf1b432fc 100644 --- a/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml +++ b/examples/machine-learning/a4-highgpu-8g/a4high-slurm-blueprint.yaml @@ -24,7 +24,7 @@ vars: # Image settings base_image: project: ubuntu-os-accelerator-images - family: ubuntu-accelerator-2204-amd64-with-nvidia-570 + image: ubuntu-accelerator-2204-amd64-with-nvidia-570-v20250712 image_build_machine_type: n2-standard-16 build_slurm_from_git_ref: 6.10.0 # Cluster env settings @@ -148,6 +148,24 @@ deployment_groups: ansible.builtin.apt: deb: "{{ cuda_repo_filename }}" state: present + # The following 2 tasks work around a temporary issue with Ubuntu + # packaging of NVIDIA 570 driver series for kernel 6.8.0-1032 + # This command ensures that any holds are removed before attempting an upgrade. + # We ignore failures in case the packages were not held. + - name: Unhold NVIDIA driver packages + ansible.builtin.command: + cmd: apt-mark unhold linux-modules-nvidia-570-server-open-gcp linux-modules-nvidia-570-server-open-6.8.0-1032-gcp + become: true + changed_when: false + failed_when: false + - name: Install latest NVIDIA driver metapackage and kernel module + ansible.builtin.apt: + name: + - linux-modules-nvidia-570-server-open-gcp + - linux-modules-nvidia-570-server-open-6.8.0-1032-gcp + state: latest + update_cache: yes + become: true - name: Reduce NVIDIA repository priority ansible.builtin.copy: dest: /etc/apt/preferences.d/cuda-repository-pin-600 @@ -237,7 +255,7 @@ deployment_groups: settings: disk_size: $(vars.disk_size_gb) machine_type: $(vars.image_build_machine_type) - source_image_family: $(vars.base_image.family) + source_image: $(vars.base_image.image) source_image_project_id: [$(vars.base_image.project)] image_family: $(vars.instance_image.family) omit_external_ip: false