Enable DRA device metadata (KEP-5304)

.gitignore

@@ -1,4 +1,5 @@
 .cache/
+.config/
 vendor/
 [._]*.sw[a-p]
 

cmd/dra-example-kubeletplugin/driver.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	coreclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/dynamic-resource-allocation/api/metadata/v1alpha1"
 	"k8s.io/dynamic-resource-allocation/kubeletplugin"
 	"k8s.io/klog/v2"
 )
@@ -56,6 +57,8 @@ func NewDriver(ctx context.Context, config *Config) (*driver, error) {
 		kubeletplugin.RegistrarDirectoryPath(config.flags.kubeletRegistrarDirectoryPath),
 		kubeletplugin.PluginDataDirectoryPath(config.DriverPluginPath()),
 		kubeletplugin.RollingUpdate(types.UID(config.flags.podUID)),
+		kubeletplugin.MetadataVersions(v1alpha1.SchemeGroupVersion),
+		kubeletplugin.EnableDeviceMetadata(config.flags.enableDeviceMetadata),
 	)
 	if err != nil {
 		return nil, err
@@ -106,12 +109,20 @@ func (d *driver) prepareResourceClaim(ctx context.Context, claim *resourceapi.Re
 	}
 	var prepared []kubeletplugin.Device
 	for _, preparedPB := range preparedPBs {
-		prepared = append(prepared, kubeletplugin.Device{
+		dev := kubeletplugin.Device{
 			Requests:     preparedPB.GetRequestNames(),
 			PoolName:     preparedPB.GetPoolName(),
 			DeviceName:   preparedPB.GetDeviceName(),
 			CDIDeviceIDs: preparedPB.GetCdiDeviceIds(),
-		})
+		}
+		if allocDev, ok := d.state.allocatable[preparedPB.GetDeviceName()]; ok && len(allocDev.Attributes) > 0 {
+			attrs := make(map[string]resourceapi.DeviceAttribute, len(allocDev.Attributes))
+			for k, v := range allocDev.Attributes {
+				attrs[string(k)] = v
+			}
+			dev.Metadata = &kubeletplugin.DeviceMetadata{Attributes: attrs}
+		}
+		prepared = append(prepared, dev)
 	}
 
 	logger.Info("Returning newly prepared devices for claim", "uid", claim.UID, "devices", prepared)

cmd/dra-example-kubeletplugin/main.go

@@ -55,6 +55,7 @@ type Flags struct {
 	podUID                        string
 	gpuPartitions                 int
 	gpuDeviceStatus               bool
+	enableDeviceMetadata          bool
 }
 
 type Config struct {
@@ -169,6 +170,13 @@ func newApp() *cli.App {
 			Destination: &flags.gpuDeviceStatus,
 			EnvVars:     []string{"GPU_DEVICE_STATUS"},
 		},
+		&cli.BoolFlag{
+			Name:        "enable-device-metadata",
+			Usage:       "Enable DRA in-container device metadata files for prepared devices.",
+			Value:       false,
+			Destination: &flags.enableDeviceMetadata,
+			EnvVars:     []string{"ENABLE_DEVICE_METADATA"},
+		},
 	}
 	cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...)
 	cliFlags = append(cliFlags, flags.loggingConfig.Flags()...)

deployments/helm/dra-example-driver/templates/kubeletplugin.yaml

@@ -92,6 +92,10 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.uid
+        {{- if .Values.kubeletPlugin.enableDeviceMetadata }}
+        - name: ENABLE_DEVICE_METADATA
+          value: {{ .Values.kubeletPlugin.enableDeviceMetadata | quote }}
+        {{- end }}
         volumeMounts:
         - name: plugins-registry
           mountPath: {{ .Values.kubeletPlugin.kubeletRegistrarDirectoryPath | quote }}

deployments/helm/dra-example-driver/values.yaml

@@ -71,6 +71,7 @@ kubeletPlugin:
   affinity: {}
   kubeletRegistrarDirectoryPath: /var/lib/kubelet/plugins_registry
   kubeletPluginsDirectoryPath: /var/lib/kubelet/plugins
+  enableDeviceMetadata: false
   containers:
     init:
       securityContext: {}

go.mod

@@ -76,6 +76,7 @@ require (
 	github.com/google/gnostic-models v0.7.1 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect
+	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/gosuri/uitable v0.0.4 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/ianlancetaylor/demangle v0.0.0-20250417193237-f615e6bd150b // indirect
@@ -93,6 +94,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/spdystream v0.5.1 // indirect
 	github.com/moby/sys/capability v0.4.0 // indirect
 	github.com/moby/term v0.5.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -149,6 +151,7 @@ require (
 	k8s.io/cli-runtime v0.36.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20260512234627-ef417d054102 // indirect
 	k8s.io/kubectl v0.36.0 // indirect
+	k8s.io/streaming v0.36.1 // indirect
 	oras.land/oras-go/v2 v2.6.0 // indirect
 	sigs.k8s.io/controller-runtime v0.24.0 // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect

go.sum

@@ -24,6 +24,8 @@ github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8
 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
 github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
 github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -168,6 +170,8 @@ github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyE
 github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
 github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
 github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
@@ -229,6 +233,8 @@ github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQ
 github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y=
+github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
 github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk=
 github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I=
 github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
@@ -464,6 +470,8 @@ k8s.io/kubectl v0.36.0 h1:hEGr8NvIm2Wjqs2Xy48Uzmvo6lpHdGKlLyMvau2gTms=
 k8s.io/kubectl v0.36.0/go.mod h1:iDe8aV5BEi45W8k+5n71I2pJ/nwE0PHDu+/2cejzYoo=
 k8s.io/kubelet v0.36.1 h1:FcHiG9wv92xerRPNxztuhYWqwS4IilOQNPxTPQewYgo=
 k8s.io/kubelet v0.36.1/go.mod h1:e6IeoCwqc2TbneCKu6P8HjmWLi7U6SOh3Pocs32iGFM=
+k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4=
+k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s=
 k8s.io/utils v0.0.0-20260507154919-ff6756f316d2 h1:wU4tMEhLGgIbLvXQb1cfN+EcM0wf7zC6CPF+C79jroc=
 k8s.io/utils v0.0.0-20260507154919-ff6756f316d2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
 oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc=

test/e2e/driver_test.go

@@ -87,6 +87,10 @@ type DriverConfig struct {
 	// attributes (e.g. uuid, model, driverVersion) into
 	// ResourceClaim.status.devices[].data.
 	GPUDeviceStatus bool
+
+	// DeviceMetadata, when true, instructs the kubelet plugin to mount
+	// per-device metadata files into containers.
+	DeviceMetadata bool
 }
 
 // installedDriver is what installDriver returns: the identity bits downstream
@@ -161,7 +165,8 @@ func buildHelmValues(cfg DriverConfig, namespace string) map[string]any {
 		"namespaceOverride": namespace,
 		"gpuDeviceStatus":   cfg.GPUDeviceStatus,
 		"kubeletPlugin": map[string]any{
-			"numDevices": cfg.NumDevices,
+			"numDevices":           cfg.NumDevices,
+			"enableDeviceMetadata": cfg.DeviceMetadata,
 		},
 		"webhook": map[string]any{
 			"enabled": cfg.WebhookEnabled,

test/e2e/e2e_setup_test.go

@@ -44,11 +44,17 @@ import (
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/restmapper"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/dynamic-resource-allocation/api/metadata"
+	"k8s.io/dynamic-resource-allocation/devicemetadata"
 )
 
 var rootDir, demoManifestsDir string
+var restConfig *rest.Config
 var clientset *kubernetes.Clientset
 var dynamicClient dynamic.Interface
 var restMapper meta.RESTMapper
@@ -94,6 +100,7 @@ var _ = BeforeSuite(func(ctx SpecContext) {
 	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
 	config, err := kubeConfig.ClientConfig()
 	Expect(err).NotTo(HaveOccurred())
+	restConfig = config
 
 	clientset, err = kubernetes.NewForConfig(config)
 	Expect(err).NotTo(HaveOccurred())
@@ -409,6 +416,97 @@ func verifyGPUAllocation(ctx context.Context, namespace, podName, containerName
 	}, checkPodLogsTimeout, checkPodLogsInterval).Should(Succeed())
 }
 
+// execInPod runs a command in a pod container and returns stdout.
+func execInPod(ctx context.Context, namespace, podName, containerName string, command []string) ([]byte, error) {
+	GinkgoHelper()
+	req := clientset.CoreV1().RESTClient().Post().
+		Resource("pods").
+		Name(podName).
+		Namespace(namespace).
+		SubResource("exec").
+		VersionedParams(&v1.PodExecOptions{
+			Container: containerName,
+			Command:   command,
+			Stdout:    true,
+			Stderr:    true,
+		}, scheme.ParameterCodec)
+
+	executor, err := remotecommand.NewSPDYExecutor(restConfig, "POST", req.URL())
+	if err != nil {
+		return nil, fmt.Errorf("create pod exec for %s/%s container %s: %w",
+			namespace, podName, containerName, err)
+	}
+
+	var stdout, stderr bytes.Buffer
+	err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{
+		Stdout: &stdout,
+		Stderr: &stderr,
+	})
+	if err != nil {
+		return stdout.Bytes(), fmt.Errorf("exec %q in %s/%s container %s failed: %w; stderr: %s",
+			strings.Join(command, " "), namespace, podName, containerName, err, stderr.String())
+	}
+	return stdout.Bytes(), nil
+}
+
+// verifyDeviceMetadata verifies the DRA device metadata file mounted into the
+// workload container for a ResourceClaimTemplate-backed claim.
+func verifyDeviceMetadata(ctx context.Context, namespace, podName, containerName string, drv installedDriver) {
+	GinkgoHelper()
+	metadataFile := metadata.ResourceClaimTemplateFilePath(drv.DriverName, "gpu", "gpu")
+	Eventually(func(g Gomega) {
+		data, err := execInPod(ctx, namespace, podName, containerName, []string{"cat", metadataFile})
+		g.Expect(err).NotTo(HaveOccurred(),
+			"Failed to read DRA device metadata file %s from pod %s/%s container %s",
+			metadataFile, namespace, podName, containerName)
+
+		var md metadata.DeviceMetadata
+		g.Expect(devicemetadata.DecodeMetadataFromStream(json.NewDecoder(bytes.NewReader(data)), &md)).To(Succeed(),
+			"Failed to decode DRA device metadata file %s from pod %s/%s container %s",
+			metadataFile, namespace, podName, containerName)
+
+		g.Expect(md.PodClaimName).NotTo(BeNil(),
+			"Device metadata file %s has no podClaimName", metadataFile)
+		g.Expect(*md.PodClaimName).To(Equal("gpu"),
+			"Device metadata file %s has unexpected podClaimName", metadataFile)
+		g.Expect(md.Requests).To(HaveLen(1),
+			"Device metadata file %s should have exactly one request", metadataFile)
+
+		request := md.Requests[0]
+		g.Expect(request.Name).To(Equal("gpu"),
+			"Device metadata file %s has unexpected request name", metadataFile)
+		g.Expect(request.Devices).To(HaveLen(1),
+			"Device metadata file %s request %s should have exactly one device",
+			metadataFile, request.Name)
+
+		device := request.Devices[0]
+		g.Expect(device.Driver).To(Equal(drv.DriverName),
+			"Device metadata file %s has unexpected device driver", metadataFile)
+		g.Expect(device.Pool).NotTo(BeEmpty(),
+			"Device metadata file %s has empty device pool", metadataFile)
+		g.Expect(device.Name).NotTo(BeEmpty(),
+			"Device metadata file %s has empty device name", metadataFile)
+		g.Expect(device.Attributes).To(HaveKey(resourceapi.QualifiedName("uuid")),
+			"Device metadata file %s device attributes are missing uuid: %v", metadataFile, device.Attributes)
+		g.Expect(device.Attributes["uuid"].StringValue).NotTo(BeNil())
+		g.Expect(*device.Attributes["uuid"].StringValue).NotTo(BeEmpty())
+		g.Expect(device.Attributes).To(HaveKey(resourceapi.QualifiedName("model")),
+			"Device metadata file %s device attributes are missing model: %v", metadataFile, device.Attributes)
+		g.Expect(device.Attributes["model"].StringValue).NotTo(BeNil())
+		g.Expect(*device.Attributes["model"].StringValue).To(Equal("LATEST-GPU-MODEL"))
+		g.Expect(device.Attributes).To(HaveKey(resourceapi.QualifiedName("driverVersion")),
+			"Device metadata file %s device attributes are missing driverVersion: %v", metadataFile, device.Attributes)
+		g.Expect(device.Attributes["driverVersion"].VersionValue).NotTo(BeNil())
+		g.Expect(*device.Attributes["driverVersion"].VersionValue).NotTo(BeEmpty())
+
+		fmt.Fprintf(GinkgoWriter, "Pod %s/%s, container %s has DRA device metadata at %s: device=%s/%s attrs uuid=%s model=%s driverVersion=%s\n",
+			namespace, podName, containerName, metadataFile, device.Pool, device.Name,
+			*device.Attributes["uuid"].StringValue,
+			*device.Attributes["model"].StringValue,
+			*device.Attributes["driverVersion"].VersionValue)
+	}, checkPodLogsTimeout, checkPodLogsInterval).Should(Succeed())
+}
+
 // verifyDRAAdminAccess verifies that DRA_ADMIN_ACCESS is set to the expected value.
 func verifyDRAAdminAccess(ctx context.Context, namespace, podName, containerName, expectedValue string) {
 	GinkgoHelper()

test/e2e/e2e_test.go

@@ -53,6 +53,23 @@ var _ = Describe("Test GPU allocation", func() {
 		}
 	})
 
+	It("should mount DRA device metadata into the workload container", func(ctx SpecContext) {
+		drv := installDriver(ctx, DriverConfig{
+			DeviceMetadata: true,
+		})
+		namespace := "basic-resourceclaimtemplate"
+		pods := []string{"pod0"}
+		containerName := "ctr0"
+		expectedGPUCount := 1
+
+		deployManifest(ctx, namespace, "basic-resourceclaimtemplate.yaml", drv)
+		checkPodsReadyAndRunning(ctx, namespace, pods)
+
+		observedGPUs := make(map[string]string)
+		verifyGPUAllocation(ctx, namespace, pods[0], containerName, expectedGPUCount, observedGPUs)
+		verifyDeviceMetadata(ctx, namespace, pods[0], containerName, drv)
+	})
+
 	It("should allocate 2 distinct GPUs to a single container", func(ctx SpecContext) {
 		drv := installDriver(ctx, DriverConfig{})
 		namespace := "basic-multiple-requests"