Fix: Thunderbolt eGPU hot-plug/unplug kernel crash support

Author: bdandy

kernel-open/common/inc/nvkms-kapi.h

@@ -660,6 +660,17 @@ struct NvKmsKapiFunctionsTable {
      */
     void (*freeDevice)(struct NvKmsKapiDevice *device);
 
+    /*!
+     * Frees a device during surprise removal (e.g., Thunderbolt eGPU unplug).
+     * This skips all hardware access and only releases kernel resources.
+     * Use this instead of freeDevice() when the GPU hardware is no longer
+     * accessible to avoid page faults and hangs.
+     *
+     * \param [in]  device  A device returned by allocateDevice().
+     *                      This function is a no-op if device is not valid.
+     */
+    void (*freeDeviceForSurpriseRemoval)(struct NvKmsKapiDevice *device);
+
     /*!
      * Grab ownership of device, ownership is required to do modeset.
      *

kernel-open/nvidia-drm/nvidia-drm-drv.c

@@ -852,6 +852,43 @@ static void nv_drm_dev_unload(struct drm_device *dev)
         return;
     }
 
+    /*
+     * During surprise removal (e.g., Thunderbolt eGPU hot-unplug),
+     * the GPU hardware is no longer accessible. Skip NVKMS calls that
+     * would access hardware to prevent page faults and crashes.
+     * Use freeDeviceForSurpriseRemoval which only releases kernel resources
+     * without attempting any hardware access.
+     */
+    if (nv_dev->inSurpriseRemoval) {
+        NV_DRM_DEV_LOG_INFO(nv_dev,
+            "Surprise removal detected, skipping hardware access");
+
+        cancel_delayed_work_sync(&nv_dev->hotplug_event_work);
+        mutex_lock(&nv_dev->lock);
+
+        atomic_set(&nv_dev->enable_event_handling, false);
+        drm_kms_helper_poll_fini(dev);
+        drm_mode_config_cleanup(dev);
+
+        pDevice = nv_dev->pDevice;
+        nv_dev->pDevice = NULL;
+
+        mutex_unlock(&nv_dev->lock);
+
+        /*
+         * Use freeDeviceForSurpriseRemoval instead of freeDevice.
+         * This skips KmsFreeDevice() and RmFreeDevice() which would try
+         * to access GPU hardware via ioctls/RM API calls and cause
+         * page faults since the GPU memory is unmapped.
+         * It only calls nvkms_close_gpu() to release the GPU reference
+         * count, allowing the eGPU to be re-initialized when reconnected.
+         */
+        if (pDevice != NULL) {
+            nvKms->freeDeviceForSurpriseRemoval(pDevice);
+        }
+        return;
+    }
+
     /* Release modeset ownership if fbdev is enabled */
 
 #if defined(NV_DRM_FBDEV_AVAILABLE)
@@ -2160,6 +2197,28 @@ static void nv_drm_dev_destroy(struct nv_drm_device *nv_dev)
     nv_drm_free(nv_dev);
 }
 
+/*
+ * Helper to get PCI device from DRM device, handling both old and new kernels.
+ * Returns NULL if not a PCI device or device not available.
+ */
+static struct pci_dev *nv_drm_get_pci_dev(struct drm_device *dev)
+{
+    if (dev == NULL) {
+        return NULL;
+    }
+
+#if defined(NV_DRM_DEVICE_HAS_PDEV)
+    return dev->pdev;
+#else
+    /* On newer kernels (5.14+), drm_device.pdev was removed.
+     * Get PCI device from the parent device. */
+    if (dev->dev != NULL && dev->dev->bus == &pci_bus_type) {
+        return to_pci_dev(dev->dev);
+    }
+    return NULL;
+#endif
+}
+
 /*
  * Unregister a single NVIDIA DRM device.
  */
@@ -2168,7 +2227,26 @@ void nv_drm_remove(NvU32 gpuId)
     struct nv_drm_device *nv_dev = nv_drm_find_and_remove_device(gpuId);
 
     if (nv_dev) {
+        struct pci_dev *pdev;
+
         NV_DRM_DEV_LOG_INFO(nv_dev, "Removing device");
+
+        /*
+         * Check if this is a surprise removal (hot-unplug) by testing
+         * if the PCI channel is offline. This happens when:
+         * - Thunderbolt eGPU is physically disconnected
+         * - GPU falls off the bus unexpectedly
+         * 
+         * For normal driver unload (rmmod), the PCI channel remains online.
+         * We only skip NVKMS hardware access during surprise removal.
+         */
+        pdev = nv_drm_get_pci_dev(nv_dev->dev);
+        if (pdev != NULL && pci_channel_offline(pdev)) {
+            NV_DRM_DEV_LOG_INFO(nv_dev,
+                "PCI channel offline - surprise removal detected");
+            nv_dev->inSurpriseRemoval = NV_TRUE;
+        }
+
         drm_dev_unplug(nv_dev->dev);
         nv_drm_dev_destroy(nv_dev);
     }

kernel-open/nvidia-drm/nvidia-drm-priv.h

@@ -146,6 +146,21 @@ struct nv_drm_device {
     NvBool subOwnershipGranted;
     NvBool hasFramebufferConsole;
 
+    /*
+     * Set to NV_TRUE for external GPUs (e.g., Thunderbolt/USB4 eGPU).
+     * External GPUs use the fast removal path to avoid hangs during
+     * both surprise removal and "safe" software-initiated disconnect.
+     */
+    NvBool isExternalGpu;
+
+    /*
+     * Set to NV_TRUE when the device is being removed due to
+     * surprise removal (e.g., Thunderbolt eGPU hot-unplug).
+     * When set, NVKMS operations that would access GPU hardware
+     * are skipped to prevent crashes from accessing unmapped memory.
+     */
+    NvBool inSurpriseRemoval;
+
     struct drm_property *nv_out_fence_property;
     struct drm_property *nv_input_colorspace_property;
 

kernel-open/nvidia-modeset/nvidia-modeset-linux.c

@@ -1204,6 +1204,27 @@ void nvkms_close_gpu(NvU32 gpuId)
     __rm_ops.free_stack(stack);
 }
 
+void nvkms_gpu_lost(NvU32 gpuId)
+{
+    /*
+     * Mark the GPU as lost in NVKMS. This prevents hardware access
+     * and cancels pending timers that might try to access the removed GPU.
+     *
+     * NOTE: We intentionally do NOT take nvkms_lock here because this function
+     * may be called from contexts that already hold the lock (e.g., during
+     * module unload). The gpuLost flag is a simple boolean that can be safely
+     * written without a lock - any racing operation will either:
+     * 1. See gpuLost=TRUE and bail out early
+     * 2. See gpuLost=FALSE but hit the 0xFFFFFFFF check when reading hardware
+     *
+     * A memory barrier ensures the write is visible to other CPUs promptly.
+     */
+    nvKmsGpuLost(gpuId);
+
+    /* Ensure gpuLost write is visible to other CPUs */
+    smp_wmb();
+}
+
 NvU32 nvkms_enumerate_gpus(nv_gpu_info_t *gpu_info)
 {
     return __rm_ops.enumerate_gpus(gpu_info);

kernel-open/nvidia-modeset/nvidia-modeset-os-interface.h

@@ -309,6 +309,12 @@ void* nvkms_get_per_open_data(int fd);
 NvBool nvkms_open_gpu(NvU32 gpuId);
 void nvkms_close_gpu(NvU32 gpuId);
 
+/*!
+ * Mark a GPU as lost (surprise removal, e.g., Thunderbolt eGPU unplug).
+ * This prevents hardware access and cancels pending timers.
+ */
+void nvkms_gpu_lost(NvU32 gpuId);
+
 
 /*!
  * Enumerate nvidia gpus.

kernel-open/nvidia-modeset/nvkms.h

@@ -88,6 +88,8 @@ void nvKmsModuleUnload(void);
 void nvKmsSuspend(NvU32 gpuId);
 void nvKmsResume(NvU32 gpuId);
 
+void nvKmsGpuLost(NvU32 gpuId);
+
 void nvKmsGetProcFiles(const nvkms_procfs_file_t **ppProcFiles);
 
 NvBool nvKmsReadConf(const char *buff, size_t size,

kernel-open/nvidia/nv-acpi.c

@@ -261,12 +261,28 @@ static void nv_acpi_notify_event(acpi_handle handle, u32 event_type, void *data)
 {
     nv_acpi_t  *pNvAcpiObject = data;
     nv_state_t *nvl = pNvAcpiObject->notifier_data;
+    nv_state_t *nv;
+
+    if (nvl == NULL)
+        return;
+
+    nv = NV_STATE_PTR(nvl);
+    if (nv == NULL)
+        return;
+
+    /*
+     * Check if we're in surprise removal before processing ACPI events.
+     * This can happen during Thunderbolt eGPU hot-unplug where the device
+     * is being removed but ACPI events are still being delivered.
+     */
+    if (nv->flags & NV_FLAG_IN_SURPRISE_REMOVAL)
+        return;
 
     /*
      * Function to handle device specific ACPI events such as display hotplug,
      * GPS and D-notifier events.
      */
-    rm_acpi_notify(pNvAcpiObject->sp, NV_STATE_PTR(nvl), event_type);
+    rm_acpi_notify(pNvAcpiObject->sp, nv, event_type);
 }
 
 void nv_acpi_register_notifier(nv_linux_state_t *nvl)

kernel-open/nvidia/nv-i2c.c

@@ -44,6 +44,15 @@ static int nv_i2c_algo_master_xfer(struct i2c_adapter *adapter, struct i2c_msg m
 #endif
     ;
 
+    /*
+     * Check if the GPU is in surprise removal (e.g., Thunderbolt unplug).
+     * If so, return immediately to avoid hanging on RPC calls to GSP.
+     */
+    if (nv_check_gpu_state(nv) != NV_OK)
+    {
+        return -ENODEV;
+    }
+
     rc = nv_kmem_cache_alloc_stack(&sp);
     if (rc != 0)
     {
@@ -93,6 +102,15 @@ static int nv_i2c_algo_smbus_xfer(
     NV_STATUS rmStatus = NV_OK;
     nvidia_stack_t *sp = NULL;
 
+    /*
+     * Check if the GPU is in surprise removal (e.g., Thunderbolt unplug).
+     * If so, return immediately to avoid hanging on RPC calls to GSP.
+     */
+    if (nv_check_gpu_state(nv) != NV_OK)
+    {
+        return -ENODEV;
+    }
+
     rc = nv_kmem_cache_alloc_stack(&sp);
     if (rc != 0)
     {
@@ -196,6 +214,15 @@ static u32 nv_i2c_algo_functionality(struct i2c_adapter *adapter)
     u32 ret = I2C_FUNC_I2C;
     nvidia_stack_t *sp = NULL;
 
+    /*
+     * Check if the GPU is in surprise removal (e.g., Thunderbolt unplug).
+     * If so, return 0 to indicate no functionality available.
+     */
+    if (nv_check_gpu_state(nv) != NV_OK)
+    {
+        return 0;
+    }
+
     if (nv_kmem_cache_alloc_stack(&sp) != 0)
     {
         return 0;

kernel-open/nvidia/nv-pci.c

@@ -27,6 +27,7 @@
 #include "nv-msi.h"
 #include "nv-hypervisor.h"
 #include "nv-reg.h"
+#include "nv-rsync.h"
 
 #if defined(NV_VGPU_KVM_BUILD)
 #include "nv-vgpu-vfio-interface.h"
@@ -1736,6 +1737,13 @@ nv_pci_remove(struct pci_dev *pci_dev)
 
     nv = NV_STATE_PTR(nvl);
 
+    /*
+     * Note: For external GPUs (eGPU via Thunderbolt), the NV_FLAG_IN_SURPRISE_REMOVAL
+     * flag is set later in the removal process - either when waiting for usage count
+     * times out, or when actual surprise removal is detected. Setting it too early
+     * can interfere with normal cleanup operations that need to acquire GPU locks.
+     */
+
 #if NV_IS_EXPORT_SYMBOL_GPL_iommu_dev_disable_feature
 #if defined(CONFIG_IOMMU_SVA) && \
     (defined(NV_IOASID_GET_PRESENT) || defined(NV_MM_PASID_DROP_PRESENT))
@@ -1773,22 +1781,34 @@ nv_pci_remove(struct pci_dev *pci_dev)
     /*
      * Sanity check: A removed device shouldn't have a non-zero usage_count.
      * For eGPU, fall off the bus along with clients active is a valid scenario.
-     * Hence skipping the sanity check for eGPU.
+     * We still wait for a short time to allow in-progress close operations
+     * to complete, but with a timeout to prevent hangs.
      */
-    if ((NV_ATOMIC_READ(nvl->usage_count) != 0) && !(nv->is_external_gpu))
+    if (NV_ATOMIC_READ(nvl->usage_count) != 0)
     {
+        /*
+         * For external GPU: wait up to 5 seconds (10 iterations * 500ms)
+         * For internal GPU: wait up to 60 seconds (120 iterations * 500ms)
+         * This prevents indefinite hangs while still allowing time for
+         * graceful cleanup of in-progress operations.
+         */
+        int max_wait_iterations = nv->is_external_gpu ? 10 : 120;
+        int wait_iterations = 0;
+
         nv_printf(NV_DBG_ERRORS,
-                  "NVRM: Attempting to remove device %04x:%02x:%02x.%x with non-zero usage count!\n",
+                  "NVRM: Attempting to remove device %04x:%02x:%02x.%x with non-zero usage count (%d)%s\n",
                   NV_PCI_DOMAIN_NUMBER(pci_dev), NV_PCI_BUS_NUMBER(pci_dev),
-                  NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn));
+                  NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn),
+                  NV_ATOMIC_READ(nvl->usage_count),
+                  nv->is_external_gpu ? " (external GPU)" : "");
 
         /*
          * We can't return from this function without corrupting state, so we wait for
-         * the usage count to go to zero.
+         * the usage count to go to zero, but with a timeout.
          */
-        while (NV_ATOMIC_READ(nvl->usage_count) != 0)
+        while ((NV_ATOMIC_READ(nvl->usage_count) != 0) &&
+               (wait_iterations < max_wait_iterations))
         {
-
             /*
              * While waiting, release the locks so that other threads can make
              * forward progress.
@@ -1797,6 +1817,7 @@ nv_pci_remove(struct pci_dev *pci_dev)
             UNLOCK_NV_LINUX_DEVICES();
 
             os_delay(500);
+            wait_iterations++;
 
             /* Re-acquire the locks before checking again */
             LOCK_NV_LINUX_DEVICES();
@@ -1815,10 +1836,32 @@ nv_pci_remove(struct pci_dev *pci_dev)
             down(&nvl->ldata_lock);
         }
 
-        nv_printf(NV_DBG_ERRORS,
-                  "NVRM: Continuing with GPU removal for device %04x:%02x:%02x.%x\n",
-                  NV_PCI_DOMAIN_NUMBER(pci_dev), NV_PCI_BUS_NUMBER(pci_dev),
-                  NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn));
+        if (NV_ATOMIC_READ(nvl->usage_count) != 0)
+        {
+            nv_printf(NV_DBG_ERRORS,
+                      "NVRM: Timeout waiting for usage count on device %04x:%02x:%02x.%x (remaining: %d). Forcing removal.\n",
+                      NV_PCI_DOMAIN_NUMBER(pci_dev), NV_PCI_BUS_NUMBER(pci_dev),
+                      NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn),
+                      NV_ATOMIC_READ(nvl->usage_count));
+            /*
+             * Force the surprise removal flag so that any remaining
+             * close operations will take the fast-path.
+             */
+            nv->flags |= NV_FLAG_IN_SURPRISE_REMOVAL;
+
+            /*
+             * Mark that we had a surprise removal so rsync cleanup
+             * warnings are suppressed during module unload.
+             */
+            nv_set_rsync_had_surprise_removal();
+        }
+        else
+        {
+            nv_printf(NV_DBG_ERRORS,
+                      "NVRM: Continuing with GPU removal for device %04x:%02x:%02x.%x\n",
+                      NV_PCI_DOMAIN_NUMBER(pci_dev), NV_PCI_BUS_NUMBER(pci_dev),
+                      NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn));
+        }
     }
 
     rm_check_for_gpu_surprise_removal(sp, nv);