Add xpumd health source support for GPU plugin (#2279)

The GPU plugin currently gets health data from the Level-Zero sidecar, which requires a privileged container. XPUMD 2.x exposes a local gRPC streaming API (WatchDeviceHealth) that provides equivalent health info without needing a privileged sidecar.

Signed-off-by: Tuomas Katila <tuomas.katila@intel.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>

Author: Copilot

cmd/gpu_plugin/README.md

@@ -19,6 +19,7 @@ Table of Contents
   * [CDI support](#cdi-support)
   * [KMD and UMD](#kmd-and-umd)
   * [Health management](#health-management)
+  * [xpumd health source](#xpumd-health-source)
   * [by-path mounting](#by-path-mounting)
   * [Issues with media workloads on multi-GPU setups](#issues-with-media-workloads-on-multi-gpu-setups)
     * [Workaround for QSV and VA-API](#workaround-for-qsv-and-va-api)
@@ -59,6 +60,7 @@ For workloads on different KMDs, see [KMD and UMD](#kmd-and-umd).
 | -enable-monitoring | - | disabled | Enable '*_monitoring' resource that provides access to all Intel GPU devices on the node, [see use](./monitoring.md) |
 | -monitoring-mode | string | single | How monitoring resources are registered: single or split |
 | -health-management | - | disabled | Enable health management by requesting data from oneAPI/Level-Zero interface. Requires [GPU Level-Zero](../gpu_levelzero/) sidecar. See [health management](#health-management) |
+| -xpumd-endpoint | string | "" | Unix socket path for xpumd health service (e.g. `/run/xpumd/intelxpuinfo.sock`). When set, xpumd is used as the health data source instead of the Level-Zero sidecar. Cannot be combined with `-health-management`. Temperature limits are specified in xpumd service configuration, not with GPU plugin flags. See [xpumd health source](#xpumd-health-source) |
 | -wsl | - | disabled | Adapt plugin to run in the WSL environment. Requires [GPU Level-Zero](../gpu_levelzero/) sidecar. |
 | -shared-dev-num | int | 1 | Number of containers that can share the same GPU device |
 | -allow-ids | string | "" | A list of PCI Device IDs that are allowed to be registered as resources. Default is empty (=all registered). Cannot be used together with `deny-ids`. |
@@ -265,6 +267,22 @@ Kubernetes Device Plugin API allows passing device's healthiness to Kubelet. By
 
 Temperature limit can be provided via the command line argument, default is 100C.
 
+### xpumd health source
+
+As an alternative to the Level-Zero sidecar, GPU plugin can obtain device health data from [Intel XPU Manager (xpumd)](https://github.com/intel/xpumanager) v2.x that provides equivalent health information without requiring a privileged sidecar container.
+
+When xpumd is running on the host it creates a unix socket (default: `/run/xpumd/intelxpuinfo.sock`). The GPU plugin connects to this socket and streams health events. A device is reported as `Unhealthy` if any health domain reports severity `WARNING` or higher.
+
+To use xpumd as the health source, deploy the plugin with the provided Kustomize overlay:
+
+```bash
+kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/gpu_plugin/overlays/xpumd?ref=<RELEASE_VERSION>'
+```
+
+The overlay mounts `/run/xpumd` from the host (read-only) into the plugin pod and passes the required `-xpumd-endpoint` and `-enable-monitoring` flags automatically.
+
+> **Note**: `-xpumd-endpoint` and (sidecar) `-health-management` flags are mutually exclusive. Sidecar specific temperature limit flags (`-temp-limit`, `-gpu-temp-limit`, `-memory-temp-limit`) are not applicable when using `xpumd` as health source.
+
 ### By-path mounting
 
 The DRM devices for the Intel GPUs register `by-path` symlinks under `/dev/dri/by-path`. For each GPU character device, there is a corresponding symlink in the by-path directory:

cmd/gpu_plugin/gpu_plugin.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io/fs"
@@ -32,6 +33,7 @@ import (
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/levelzeroservice"
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/xpumdservice"
 	gpulevelzero "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/levelzero"
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 	cdispec "tags.cncf.io/container-device-interface/specs-go"
@@ -71,6 +73,9 @@ const (
 
 	// Period of device scans.
 	scanPeriod = 5 * time.Second
+
+	// Default limit for temperatures.
+	defaultTempLimit = 100
 )
 
 type cliOptions struct {
@@ -79,6 +84,7 @@ type cliOptions struct {
 	denyIDs                   string
 	bypathMount               string
 	monitoringMode            string
+	xpumdEndpoint             string
 	sharedDevNum              int
 	globalTempLimit           int
 	memoryTempLimit           int
@@ -88,6 +94,20 @@ type cliOptions struct {
 	healthManagement          bool
 }
 
+type argError struct {
+	msg string
+}
+
+func (e argError) Error() string {
+	return fmt.Sprintf("argument error: %s", e.msg)
+}
+
+func newArgError(msg string) error {
+	return argError{
+		msg: msg,
+	}
+}
+
 func validatePCIDeviceIDs(pciIDList string) error {
 	if pciIDList == "" {
 		return nil
@@ -98,11 +118,11 @@ func validatePCIDeviceIDs(pciIDList string) error {
 	for id := range strings.SplitSeq(pciIDList, ",") {
 		id = strings.TrimSpace(id)
 		if id == "" {
-			return os.ErrNotExist
+			return newArgError("empty PCI ID")
 		}
 
 		if !r.MatchString(id) {
-			return os.ErrInvalid
+			return newArgError("invalid PCI ID: " + id)
 		}
 	}
 
@@ -191,6 +211,7 @@ type devicePlugin struct {
 	scanResources chan bool
 
 	levelzeroService levelzeroservice.LevelzeroService
+	xpumdService     xpumdservice.XpumdService
 
 	sysfsDrmDir    string
 	devFsRoot      string
@@ -255,15 +276,33 @@ func logHealthStatusChange(card, newStatus string, statuses map[string]string) {
 	}
 }
 
-func (dp *devicePlugin) healthStatusForCard(cardPath string) string {
-	if dp.levelzeroService == nil {
-		return pluginapi.Healthy
-	}
-
+// bdfForCard resolves the PCI BDF address for a card sysfs path by following
+// the "device" symlink.  It returns ("", false) when the link cannot be read.
+func bdfForCard(cardPath string) (string, bool) {
 	link, err := os.Readlink(filepath.Join(cardPath, "device"))
 	if err != nil {
 		klog.Warning("couldn't read device link for", cardPath)
 
+		return "", false
+	}
+
+	return filepath.Base(link), true
+}
+
+func (dp *devicePlugin) healthStatusForCard(cardPath string) string {
+	if dp.xpumdService != nil {
+		return dp.healthStatusForCardXpumd(cardPath)
+	} else if dp.levelzeroService != nil {
+		return dp.healthStatusForCardLZ(cardPath)
+	}
+
+	return pluginapi.Healthy
+}
+
+// healthStatusForCardLZ checks device health using the Level-Zero service.
+func (dp *devicePlugin) healthStatusForCardLZ(cardPath string) string {
+	bdfAddr, ok := bdfForCard(cardPath)
+	if !ok {
 		return pluginapi.Healthy
 	}
 
@@ -272,8 +311,6 @@ func (dp *devicePlugin) healthStatusForCard(cardPath string) string {
 	// Check status changes after the function exits
 	defer func() { logHealthStatusChange(cardPath, health, dp.healthStatuses) }()
 
-	bdfAddr := filepath.Base(link)
-
 	dh, err := dp.levelzeroService.GetDeviceHealth(bdfAddr)
 	if err != nil {
 		klog.Warningf("Device health retrieval failed: %v", err)
@@ -311,6 +348,34 @@ func (dp *devicePlugin) healthStatusForCard(cardPath string) string {
 	return health
 }
 
+// healthStatusForCardXpumd checks device health using the xpumd service.
+func (dp *devicePlugin) healthStatusForCardXpumd(cardPath string) string {
+	bdfAddr, ok := bdfForCard(cardPath)
+	if !ok {
+		return pluginapi.Healthy
+	}
+
+	health := pluginapi.Healthy
+
+	// Check status changes after the function exits
+	defer func() { logHealthStatusChange(cardPath, health, dp.healthStatuses) }()
+
+	healthy, err := dp.xpumdService.GetDeviceHealth(bdfAddr)
+	if err != nil {
+		klog.Warningf("xpumd device health retrieval failed: %v", err)
+
+		return health
+	}
+
+	klog.V(4).Infof("xpumd health for %s: Healthy=%t", bdfAddr, healthy)
+
+	if !healthy {
+		health = pluginapi.Unhealthy
+	}
+
+	return health
+}
+
 // Implement the PreferredAllocator interface.
 func (dp *devicePlugin) GetPreferredAllocation(rqt *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) {
 	response := &pluginapi.PreferredAllocationResponse{}
@@ -706,23 +771,67 @@ func (dp *devicePlugin) Allocate(request *pluginapi.AllocateRequest) (*pluginapi
 	return nil, &dpapi.UseDefaultMethodError{}
 }
 
-func checkAllowDenyOptions(opts cliOptions) bool {
+func checkBasics(opts cliOptions) error {
+	if opts.sharedDevNum < 1 {
+		return newArgError("the number of containers sharing the same GPU must greater than zero")
+	}
+
+	var str = opts.preferredAllocationPolicy
+	if !(str == "balanced" || str == "packed" || str == "none") {
+		return newArgError("invalid value for preferredAllocationPolicy, the valid values: balanced, packed, none")
+	}
+
 	if len(opts.allowIDs) > 0 && len(opts.denyIDs) > 0 {
-		klog.Error("Cannot use both allow-ids and deny-ids options at the same time. Please use only one of them.")
-		return false
+		return newArgError("cannot use both allow-ids and deny-ids options at the same time")
 	}
 
 	if err := validatePCIDeviceIDs(opts.allowIDs); err != nil {
-		klog.Error("Failed to validate allow-ids: ", err)
-		return false
+		return fmt.Errorf("failed to validate allow-ids: %w", err)
 	}
 
 	if err := validatePCIDeviceIDs(opts.denyIDs); err != nil {
-		klog.Error("Failed to validate deny-ids: ", err)
-		return false
+		return fmt.Errorf("failed to validate deny-ids: %w", err)
 	}
 
-	return true
+	switch opts.monitoringMode {
+	case monitoringModeSingle:
+	case monitoringModeSplit:
+	default:
+		return newArgError(fmt.Sprintf("invalid value for monitoring-mode, valid values: %s, %s",
+			monitoringModeSplit, monitoringModeSingle))
+	}
+
+	return nil
+}
+
+func checkArgs(opts cliOptions) error {
+	if err := checkBasics(opts); err != nil {
+		return fmt.Errorf("%w", err)
+	}
+
+	if opts.wslScan {
+		if opts.enableMonitoring {
+			return newArgError("monitoring is not supported within WSL.")
+		}
+
+		if opts.healthManagement || opts.xpumdEndpoint != "" {
+			return newArgError("health management is not supported within WSL.")
+		}
+	}
+
+	if opts.healthManagement && opts.xpumdEndpoint != "" {
+		return newArgError("cannot use both Level-Zero sidecar and xpumd for health management.")
+	}
+
+	if opts.xpumdEndpoint != "" {
+		if opts.globalTempLimit != defaultTempLimit ||
+			opts.gpuTempLimit != defaultTempLimit ||
+			opts.memoryTempLimit != defaultTempLimit {
+			return newArgError("temperature limits do not work with xpumd health source")
+		}
+	}
+
+	return nil
 }
 
 func main() {
@@ -734,69 +843,57 @@ func main() {
 	flag.StringVar(&prefix, "prefix", "", "Prefix for devfs & sysfs paths")
 	flag.BoolVar(&opts.enableMonitoring, "enable-monitoring", false, "whether to enable monitoring (= all GPUs) resource(s). See also --monitoring-mode")
 	flag.StringVar(&opts.monitoringMode, "monitoring-mode", monitoringModeSingle, "monitoring resource mode when --enable-monitoring is set: single (combined gpu.intel.com/monitoring resource) or split (per-driver i915_monitoring/xe_monitoring resources)")
-	flag.BoolVar(&opts.healthManagement, "health-management", false, "enable GPU health management")
+	flag.BoolVar(&opts.healthManagement, "health-management", false, "enable Level-Zero sidecar based GPU health management")
+	flag.StringVar(&opts.xpumdEndpoint, "xpumd-endpoint", "", "enable xpumd based health management. Argument is unix socket path for the xpumd health service (e.g. /run/xpumd/intelxpuinfo.sock). When set, health data is retrieved from xpumd")
 	flag.StringVar(&opts.bypathMount, "bypath", bypathOptionSingle, "DRI device 'by-path/' directory mounting options: single, none, all. Default: single")
 	flag.BoolVar(&opts.wslScan, "wsl", false, "scan for / use WSL devices")
-	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device")
-	flag.IntVar(&opts.globalTempLimit, "temp-limit", 100, "Global temperature limit at which device is marked unhealthy")
-	flag.IntVar(&opts.gpuTempLimit, "gpu-temp-limit", 100, "GPU temperature limit at which device is marked unhealthy")
-	flag.IntVar(&opts.memoryTempLimit, "memory-temp-limit", 100, "Memory temperature limit at which device is marked unhealthy")
+	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device.")
+	flag.IntVar(&opts.globalTempLimit, "temp-limit", defaultTempLimit, "Global temperature limit at which device is marked unhealthy. Use with health-managmement.")
+	flag.IntVar(&opts.gpuTempLimit, "gpu-temp-limit", defaultTempLimit, "GPU temperature limit at which device is marked unhealthy. Use with health-managmement.")
+	flag.IntVar(&opts.memoryTempLimit, "memory-temp-limit", defaultTempLimit, "Memory temperature limit at which device is marked unhealthy. Use with health-managmement.")
 	flag.StringVar(&opts.preferredAllocationPolicy, "allocation-policy", "none", "modes of allocating GPU devices: balanced, packed and none")
 	flag.StringVar(&opts.allowIDs, "allow-ids", "", "comma-separated list of device IDs to allow (e.g. 0x49c5,0x49c6)")
 	flag.StringVar(&opts.denyIDs, "deny-ids", "", "comma-separated list of device IDs to deny (e.g. 0x49c5,0x49c6)")
 
 	flag.Parse()
 
-	if opts.sharedDevNum < 1 {
-		klog.Error("The number of containers sharing the same GPU must greater than zero")
-		os.Exit(1)
-	}
-
-	var str = opts.preferredAllocationPolicy
-	if !(str == "balanced" || str == "packed" || str == "none") {
-		klog.Error("invalid value for preferredAllocationPolicy, the valid values: balanced, packed, none")
-		os.Exit(1)
-	}
+	klog.V(1).Infof("GPU device plugin started with %s preferred allocation policy", opts.preferredAllocationPolicy)
 
-	if !checkAllowDenyOptions(opts) {
-		klog.Error("Invalid allow/deny options.")
+	plugin := newDevicePlugin(prefix+sysFsRoot, prefix+devFsRoot, opts)
 
-		os.Exit(1)
+	if err := checkArgs(plugin.options); err != nil {
+		klog.Fatal("Argument check failed: ", err)
 	}
 
-	switch opts.monitoringMode {
-	case monitoringModeSingle:
-	case monitoringModeSplit:
-	default:
-		klog.Fatalf("invalid value for monitoring-mode, valid values: %s, %s", monitoringModeSplit, monitoringModeSingle)
-	}
+	// Setup xpumd service if enabled
+	setupXpumdService(plugin)
+	// Setup Level-Zero service if enabled
+	setupLevelZeroService(plugin)
 
-	klog.V(1).Infof("GPU device plugin started with %s preferred allocation policy", opts.preferredAllocationPolicy)
-
-	plugin := newDevicePlugin(prefix+sysFsRoot, prefix+devFsRoot, opts)
+	manager := dpapi.NewManager(namespace, plugin)
+	manager.Run()
+}
 
-	if plugin.options.wslScan {
-		klog.Info("WSL mode requested")
+func setupLevelZeroService(plugin *devicePlugin) {
+	if !plugin.options.healthManagement && !plugin.options.wslScan {
+		return
+	}
 
-		if plugin.options.enableMonitoring {
-			klog.Error("Monitoring is not supported within WSL. Please disable monitoring.")
+	klog.Info("levelzero service requested: ", gpulevelzero.DefaultUnixSocketPath)
 
-			os.Exit(1)
-		}
+	plugin.levelzeroService = levelzeroservice.NewLevelzero(gpulevelzero.DefaultUnixSocketPath)
 
-		if plugin.options.healthManagement {
-			klog.Error("Health management is not supported within WSL. Please disable health management.")
+	go plugin.levelzeroService.Run(true)
+}
 
-			os.Exit(1)
-		}
+func setupXpumdService(plugin *devicePlugin) {
+	if plugin.options.xpumdEndpoint == "" {
+		return
 	}
 
-	if plugin.options.healthManagement || plugin.options.wslScan {
-		plugin.levelzeroService = levelzeroservice.NewLevelzero(gpulevelzero.DefaultUnixSocketPath)
+	klog.Info("xpumd health source requested: ", plugin.options.xpumdEndpoint)
 
-		go plugin.levelzeroService.Run(true)
-	}
+	plugin.xpumdService = xpumdservice.NewXpumd(plugin.options.xpumdEndpoint)
 
-	manager := dpapi.NewManager(namespace, plugin)
-	manager.Run()
+	go plugin.xpumdService.Run(context.Background())
 }