[Azure] Downgrade Linux Kernel to 6.8 as a workaround to install Grid driver

Author: peterschmidt85

scripts/packer/azure-image-grid.json

@@ -24,7 +24,6 @@
     "image_publisher": "canonical",
     "image_offer": "ubuntu-24_04-lts",
     "image_sku": "server",
-    "image_version": "24.04.202509170",
     "azure_tags": {
         "Name": "DSTACK-GRID"
     },
@@ -64,6 +63,15 @@
         "./install-docker.sh --version {{user `docker_version`}}"
       ]
     },
+    {
+      "type": "shell",
+      "script": "provisioners/downgrade-azure-kernel.sh"
+    },
+    {
+      "type": "shell",
+      "inline": ["sudo reboot"],
+      "expect_disconnect": true
+    },
     {
       "type": "shell",
       "script": "provisioners/install-nvidia-grid-driver-for-azure.sh"

scripts/packer/provisioners/downgrade-azure-kernel.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# based on https://learn.microsoft.com/en-us/azure/virtual-machines/extensions/hpccompute-gpu-linux#known-issues
+# this is a temporary solution only required until the issue is fixed
+
+set -e
+
+# Install 6.8 kernel
+sudo apt-get update
+sudo DEBIAN_FRONTEND=noninteractive sudo apt install linux-image-6.8.0-1015-azure linux-headers-6.8.0-1015-azure -y
+
+# Update the Grub entry name
+sudo grub_entry_name=$(sudo grep -Po "menuentry '\KUbuntu, with Linux 6.8[^(']+" /boot/grub/grub.cfg | sort -V | head -1)
+sudo sed -i "s/^\s*GRUB_DEFAULT=.*$/GRUB_DEFAULT='Advanced options for Ubuntu>$grub_entry_name'/" /etc/default/grub
+sudo update-grub
+
+# Disable the kernel package upgrade
+sudo apt-mark hold $(dpkg --get-selections | grep -Po "^linux[^\t]+${grub_entry_name##* }")