diff --git a/internal/controllers/device_plugin_reconciler.go b/internal/controllers/device_plugin_reconciler.go index 973c1a543..c4d29e81a 100644 --- a/internal/controllers/device_plugin_reconciler.go +++ b/internal/controllers/device_plugin_reconciler.go @@ -454,8 +454,8 @@ func (dsci *daemonSetCreatorImpl) setDevicePluginAsDesired( Finalizers: []string{constants.NodeLabelerFinalizer}, }, Spec: v1.PodSpec{ - InitContainers: generatePodContainerSpec(mod.Spec.DevicePlugin.InitContainer, "device-plugin-init", nil), - Containers: generatePodContainerSpec(&mod.Spec.DevicePlugin.Container, "device-plugin", containerVolumeMounts), + InitContainers: generatePodContainerSpec(mod.Spec.DevicePlugin.InitContainer, "device-plugin-init", nil, nil, nil), + Containers: generatePodContainerSpec(&mod.Spec.DevicePlugin.Container, "device-plugin", containerVolumeMounts, nil, nil), PriorityClassName: "system-node-critical", ImagePullSecrets: getPodPullSecrets(mod.Spec.ImageRepoSecret), NodeSelector: nodeSelector, @@ -470,7 +470,13 @@ func (dsci *daemonSetCreatorImpl) setDevicePluginAsDesired( return controllerutil.SetControllerReference(mod, ds, dsci.scheme) } -func generatePodContainerSpec(containerSpec *kmmv1beta1.DevicePluginContainerSpec, containerName string, presetVolumeMounts []v1.VolumeMount) []v1.Container { +func generatePodContainerSpec( + containerSpec *kmmv1beta1.DevicePluginContainerSpec, + containerName string, + presetVolumeMounts []v1.VolumeMount, + presetEnv []v1.EnvVar, + livenessProbe *v1.Probe, +) []v1.Container { if containerSpec == nil { return nil } @@ -478,13 +484,14 @@ func generatePodContainerSpec(containerSpec *kmmv1beta1.DevicePluginContainerSpe { Args: containerSpec.Args, Command: containerSpec.Command, - Env: containerSpec.Env, + Env: append(presetEnv, containerSpec.Env...), Name: containerName, Image: containerSpec.Image, ImagePullPolicy: containerSpec.ImagePullPolicy, Resources: containerSpec.Resources, SecurityContext: &v1.SecurityContext{Privileged: ptr.To(true)}, - VolumeMounts: append(containerSpec.VolumeMounts, presetVolumeMounts...), + VolumeMounts: append(presetVolumeMounts, containerSpec.VolumeMounts...), + LivenessProbe: livenessProbe, }, } } diff --git a/internal/controllers/device_plugin_reconciler_test.go b/internal/controllers/device_plugin_reconciler_test.go index b3e20588a..8e4c6b0cf 100644 --- a/internal/controllers/device_plugin_reconciler_test.go +++ b/internal/controllers/device_plugin_reconciler_test.go @@ -907,11 +907,11 @@ var _ = Describe("DevicePluginReconciler_setDevicePluginAsDesired", func() { Privileged: ptr.To(true), }, VolumeMounts: []v1.VolumeMount{ - dpVolMount, { Name: "kubelet-device-plugins", MountPath: "/var/lib/kubelet/device-plugins", }, + dpVolMount, }, }, }, diff --git a/internal/controllers/dra_reconciler.go b/internal/controllers/dra_reconciler.go index 9cd4d43b7..12389c397 100644 --- a/internal/controllers/dra_reconciler.go +++ b/internal/controllers/dra_reconciler.go @@ -32,6 +32,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -48,6 +49,10 @@ const ( kubeletPluginsPath = "/var/lib/kubelet/plugins/" kubeletPluginsRegistryVolumeName = "kubelet-plugins-registry" kubeletPluginsRegistryPath = "/var/lib/kubelet/plugins_registry/" + cdiVolumeName = "cdi" + cdiPath = "/var/run/cdi" + + draHealthcheckPort = 51515 ) type DRAReconciler struct { @@ -438,9 +443,64 @@ func (dsci *draDaemonSetCreatorImpl) setDRAAsDesired( }, } + cdiVolume := v1.Volume{ + Name: cdiVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: cdiPath, + Type: &hostPathDirOrCreate, + }, + }, + } + containerVolumeMounts := []v1.VolumeMount{ {Name: kubeletPluginsVolumeName, MountPath: kubeletPluginsPath}, {Name: kubeletPluginsRegistryVolumeName, MountPath: kubeletPluginsRegistryPath}, + {Name: cdiVolumeName, MountPath: cdiPath}, + } + + presetEnv := []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "POD_UID", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.uid"}, + }, + }, + { + Name: "CDI_ROOT", + Value: cdiPath, + }, + { + Name: "KUBELET_REGISTRAR_DIRECTORY_PATH", + Value: kubeletPluginsRegistryPath, + }, + { + Name: "KUBELET_PLUGINS_DIRECTORY_PATH", + Value: kubeletPluginsPath, + }, + { + Name: "HEALTHCHECK_PORT", + Value: fmt.Sprintf("%d", draHealthcheckPort), + }, + } + + draLivenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + GRPC: &v1.GRPCAction{ + Port: draHealthcheckPort, + Service: ptr.To("liveness"), + }, + }, + InitialDelaySeconds: 30, + PeriodSeconds: 10, + TimeoutSeconds: 5, + FailureThreshold: 3, } standardLabels := map[string]string{ @@ -470,13 +530,14 @@ func (dsci *draDaemonSetCreatorImpl) setDRAAsDesired( Finalizers: []string{constants.NodeLabelerFinalizer}, }, Spec: v1.PodSpec{ - InitContainers: generatePodContainerSpec(mod.Spec.DRA.InitContainer, "dra-init", nil), - Containers: generatePodContainerSpec(&mod.Spec.DRA.Container, "dra", containerVolumeMounts), + InitContainers: generatePodContainerSpec(mod.Spec.DRA.InitContainer, "dra-init", nil, nil, nil), + Containers: generatePodContainerSpec(&mod.Spec.DRA.Container, "dra", containerVolumeMounts, presetEnv, draLivenessProbe), PriorityClassName: "system-node-critical", + HostNetwork: true, ImagePullSecrets: getPodPullSecrets(mod.Spec.ImageRepoSecret), NodeSelector: nodeSelector, ServiceAccountName: mod.Spec.DRA.ServiceAccountName, - Volumes: append([]v1.Volume{pluginsVolume, registryVolume}, mod.Spec.DRA.Volumes...), + Volumes: append([]v1.Volume{pluginsVolume, registryVolume, cdiVolume}, mod.Spec.DRA.Volumes...), Tolerations: mod.Spec.Tolerations, AutomountServiceAccountToken: mod.Spec.DRA.AutomountServiceAccountToken, }, diff --git a/internal/controllers/dra_reconciler_test.go b/internal/controllers/dra_reconciler_test.go index 1d767c962..981008258 100644 --- a/internal/controllers/dra_reconciler_test.go +++ b/internal/controllers/dra_reconciler_test.go @@ -479,10 +479,11 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() { err := dsc.setDRAAsDesired(context.Background(), &ds, &mod) Expect(err).NotTo(HaveOccurred()) - Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(4)) Expect(ds.Spec.Template.Spec.Volumes[0].Name).To(Equal("kubelet-plugins")) Expect(ds.Spec.Template.Spec.Volumes[1].Name).To(Equal("kubelet-plugins-registry")) - Expect(ds.Spec.Template.Spec.Volumes[2]).To(Equal(vol)) + Expect(ds.Spec.Template.Spec.Volumes[2].Name).To(Equal("cdi")) + Expect(ds.Spec.Template.Spec.Volumes[3]).To(Equal(vol)) }) DescribeTable("should work as expected", @@ -614,6 +615,25 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() { hostPathDirOrCreate := v1.HostPathDirectoryOrCreate hostPathDir := v1.HostPathDirectory + presetEnv := []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "POD_UID", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.uid"}, + }, + }, + {Name: "CDI_ROOT", Value: "/var/run/cdi"}, + {Name: "KUBELET_REGISTRAR_DIRECTORY_PATH", Value: "/var/lib/kubelet/plugins_registry/"}, + {Name: "KUBELET_PLUGINS_DIRECTORY_PATH", Value: "/var/lib/kubelet/plugins/"}, + {Name: "HEALTHCHECK_PORT", Value: "51515"}, + } + expected := appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: dsName, @@ -643,7 +663,7 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() { { Args: args, Command: command, - Env: env, + Env: append(presetEnv, env...), Image: draImage, ImagePullPolicy: ipp, Name: "dra", @@ -652,7 +672,6 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() { Privileged: ptr.To(true), }, VolumeMounts: []v1.VolumeMount{ - draVolMount, { Name: "kubelet-plugins", MountPath: "/var/lib/kubelet/plugins/", @@ -661,6 +680,23 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() { Name: "kubelet-plugins-registry", MountPath: "/var/lib/kubelet/plugins_registry/", }, + { + Name: "cdi", + MountPath: "/var/run/cdi", + }, + draVolMount, + }, + LivenessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + GRPC: &v1.GRPCAction{ + Port: 51515, + Service: ptr.To("liveness"), + }, + }, + InitialDelaySeconds: 30, + PeriodSeconds: 10, + TimeoutSeconds: 5, + FailureThreshold: 3, }, }, }, @@ -669,6 +705,7 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() { utils.GetKernelModuleReadyNodeLabel(namespace, draModuleName): "", }, PriorityClassName: "system-node-critical", + HostNetwork: true, ServiceAccountName: serviceAccountName, Volumes: []v1.Volume{ { @@ -689,6 +726,15 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() { }, }, }, + { + Name: "cdi", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/run/cdi", + Type: &hostPathDirOrCreate, + }, + }, + }, draVol, }, Tolerations: []v1.Toleration{testToleration},