Skip to content

Commit bcbf661

Browse files
yevgeny-shnaidmanopenshift-merge-bot[bot]
yevgeny-shnaidman
authored and
openshift-merge-bot[bot]
committed
Adding dra-kubelet communication volumes
This commit add mapping of a number of volumes and envs that are necessary for DRA driver <--> kubelet communication Upstream-Commit: 756ef27198310a370b992b27dc149506e4966f49
1 parent edbb3a3 commit bcbf661

4 files changed

Lines changed: 127 additions & 13 deletions

File tree

‎internal/controllers/device_plugin_reconciler.go‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -500,8 +500,8 @@ func (dsci *daemonSetCreatorImpl) setDevicePluginAsDesired(
500500
Finalizers: []string{constants.NodeLabelerFinalizer},
501501
},
502502
Spec: v1.PodSpec{
503-
InitContainers: generatePodContainerSpec(mod.Spec.DevicePlugin.InitContainer, "device-plugin-init", nil),
504-
Containers: generatePodContainerSpec(&mod.Spec.DevicePlugin.Container, "device-plugin", containerVolumeMounts),
503+
InitContainers: generatePodContainerSpec(mod.Spec.DevicePlugin.InitContainer, "device-plugin-init", nil, nil, nil),
504+
Containers: generatePodContainerSpec(&mod.Spec.DevicePlugin.Container, "device-plugin", containerVolumeMounts, nil, nil),
505505
PriorityClassName: "system-node-critical",
506506
ImagePullSecrets: getPodPullSecrets(mod.Spec.ImageRepoSecret),
507507
NodeSelector: nodeSelector,
@@ -516,21 +516,28 @@ func (dsci *daemonSetCreatorImpl) setDevicePluginAsDesired(
516516
return controllerutil.SetControllerReference(mod, ds, dsci.scheme)
517517
}
518518

519-
func generatePodContainerSpec(containerSpec *kmmv1beta1.DevicePluginContainerSpec, containerName string, presetVolumeMounts []v1.VolumeMount) []v1.Container {
519+
func generatePodContainerSpec(
520+
containerSpec *kmmv1beta1.DevicePluginContainerSpec,
521+
containerName string,
522+
presetVolumeMounts []v1.VolumeMount,
523+
presetEnv []v1.EnvVar,
524+
livenessProbe *v1.Probe,
525+
) []v1.Container {
520526
if containerSpec == nil {
521527
return nil
522528
}
523529
return []v1.Container{
524530
{
525531
Args: containerSpec.Args,
526532
Command: containerSpec.Command,
527-
Env: containerSpec.Env,
533+
Env: append(presetEnv, containerSpec.Env...),
528534
Name: containerName,
529535
Image: containerSpec.Image,
530536
ImagePullPolicy: containerSpec.ImagePullPolicy,
531537
Resources: containerSpec.Resources,
532538
SecurityContext: &v1.SecurityContext{Privileged: ptr.To(true)},
533-
VolumeMounts: append(containerSpec.VolumeMounts, presetVolumeMounts...),
539+
VolumeMounts: append(presetVolumeMounts, containerSpec.VolumeMounts...),
540+
LivenessProbe: livenessProbe,
534541
},
535542
}
536543
}

‎internal/controllers/device_plugin_reconciler_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -952,11 +952,11 @@ var _ = Describe("DevicePluginReconciler_setDevicePluginAsDesired", func() {
952952
Privileged: ptr.To(true),
953953
},
954954
VolumeMounts: []v1.VolumeMount{
955-
dpVolMount,
956955
{
957956
Name: "kubelet-device-plugins",
958957
MountPath: "/var/lib/kubelet/device-plugins",
959958
},
959+
dpVolMount,
960960
},
961961
},
962962
},

‎internal/controllers/dra_reconciler.go‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
apierrors "k8s.io/apimachinery/pkg/api/errors"
3434
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3535
"k8s.io/apimachinery/pkg/runtime"
36+
"k8s.io/utils/ptr"
3637
ctrl "sigs.k8s.io/controller-runtime"
3738
"sigs.k8s.io/controller-runtime/pkg/builder"
3839
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -49,6 +50,10 @@ const (
4950
kubeletPluginsPath = "/var/lib/kubelet/plugins/"
5051
kubeletPluginsRegistryVolumeName = "kubelet-plugins-registry"
5152
kubeletPluginsRegistryPath = "/var/lib/kubelet/plugins_registry/"
53+
cdiVolumeName = "cdi"
54+
cdiPath = "/var/run/cdi"
55+
56+
draHealthcheckPort = 51515
5257
)
5358

5459
type DRAReconciler struct {
@@ -465,9 +470,64 @@ func (dsci *draDaemonSetCreatorImpl) setDRAAsDesired(
465470
},
466471
}
467472

473+
cdiVolume := v1.Volume{
474+
Name: cdiVolumeName,
475+
VolumeSource: v1.VolumeSource{
476+
HostPath: &v1.HostPathVolumeSource{
477+
Path: cdiPath,
478+
Type: &hostPathDirOrCreate,
479+
},
480+
},
481+
}
482+
468483
containerVolumeMounts := []v1.VolumeMount{
469484
{Name: kubeletPluginsVolumeName, MountPath: kubeletPluginsPath},
470485
{Name: kubeletPluginsRegistryVolumeName, MountPath: kubeletPluginsRegistryPath},
486+
{Name: cdiVolumeName, MountPath: cdiPath},
487+
}
488+
489+
presetEnv := []v1.EnvVar{
490+
{
491+
Name: "NODE_NAME",
492+
ValueFrom: &v1.EnvVarSource{
493+
FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"},
494+
},
495+
},
496+
{
497+
Name: "POD_UID",
498+
ValueFrom: &v1.EnvVarSource{
499+
FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.uid"},
500+
},
501+
},
502+
{
503+
Name: "CDI_ROOT",
504+
Value: cdiPath,
505+
},
506+
{
507+
Name: "KUBELET_REGISTRAR_DIRECTORY_PATH",
508+
Value: kubeletPluginsRegistryPath,
509+
},
510+
{
511+
Name: "KUBELET_PLUGINS_DIRECTORY_PATH",
512+
Value: kubeletPluginsPath,
513+
},
514+
{
515+
Name: "HEALTHCHECK_PORT",
516+
Value: fmt.Sprintf("%d", draHealthcheckPort),
517+
},
518+
}
519+
520+
draLivenessProbe := &v1.Probe{
521+
ProbeHandler: v1.ProbeHandler{
522+
GRPC: &v1.GRPCAction{
523+
Port: draHealthcheckPort,
524+
Service: ptr.To("liveness"),
525+
},
526+
},
527+
InitialDelaySeconds: 30,
528+
PeriodSeconds: 10,
529+
TimeoutSeconds: 5,
530+
FailureThreshold: 3,
471531
}
472532

473533
standardLabels := map[string]string{
@@ -505,13 +565,14 @@ func (dsci *draDaemonSetCreatorImpl) setDRAAsDesired(
505565
Finalizers: []string{constants.NodeLabelerFinalizer},
506566
},
507567
Spec: v1.PodSpec{
508-
InitContainers: generatePodContainerSpec(mod.Spec.DRA.InitContainer, "dra-init", nil),
509-
Containers: generatePodContainerSpec(&mod.Spec.DRA.Container, "dra", containerVolumeMounts),
568+
InitContainers: generatePodContainerSpec(mod.Spec.DRA.InitContainer, "dra-init", nil, nil, nil),
569+
Containers: generatePodContainerSpec(&mod.Spec.DRA.Container, "dra", containerVolumeMounts, presetEnv, draLivenessProbe),
510570
PriorityClassName: "system-node-critical",
571+
HostNetwork: true,
511572
ImagePullSecrets: getPodPullSecrets(mod.Spec.ImageRepoSecret),
512573
NodeSelector: nodeSelector,
513574
ServiceAccountName: mod.Spec.DRA.ServiceAccountName,
514-
Volumes: append([]v1.Volume{pluginsVolume, registryVolume}, mod.Spec.DRA.Volumes...),
575+
Volumes: append([]v1.Volume{pluginsVolume, registryVolume, cdiVolume}, mod.Spec.DRA.Volumes...),
515576
Tolerations: mod.Spec.Tolerations,
516577
AutomountServiceAccountToken: mod.Spec.DRA.AutomountServiceAccountToken,
517578
},

‎internal/controllers/dra_reconciler_test.go‎

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -491,10 +491,11 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() {
491491

492492
err := dsc.setDRAAsDesired(context.Background(), &ds, &mod)
493493
Expect(err).NotTo(HaveOccurred())
494-
Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(3))
494+
Expect(ds.Spec.Template.Spec.Volumes).To(HaveLen(4))
495495
Expect(ds.Spec.Template.Spec.Volumes[0].Name).To(Equal("kubelet-plugins"))
496496
Expect(ds.Spec.Template.Spec.Volumes[1].Name).To(Equal("kubelet-plugins-registry"))
497-
Expect(ds.Spec.Template.Spec.Volumes[2]).To(Equal(vol))
497+
Expect(ds.Spec.Template.Spec.Volumes[2].Name).To(Equal("cdi"))
498+
Expect(ds.Spec.Template.Spec.Volumes[3]).To(Equal(vol))
498499
})
499500

500501
DescribeTable("should work as expected",
@@ -634,6 +635,25 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() {
634635
hostPathDirOrCreate := v1.HostPathDirectoryOrCreate
635636
hostPathDir := v1.HostPathDirectory
636637

638+
presetEnv := []v1.EnvVar{
639+
{
640+
Name: "NODE_NAME",
641+
ValueFrom: &v1.EnvVarSource{
642+
FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"},
643+
},
644+
},
645+
{
646+
Name: "POD_UID",
647+
ValueFrom: &v1.EnvVarSource{
648+
FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.uid"},
649+
},
650+
},
651+
{Name: "CDI_ROOT", Value: "/var/run/cdi"},
652+
{Name: "KUBELET_REGISTRAR_DIRECTORY_PATH", Value: "/var/lib/kubelet/plugins_registry/"},
653+
{Name: "KUBELET_PLUGINS_DIRECTORY_PATH", Value: "/var/lib/kubelet/plugins/"},
654+
{Name: "HEALTHCHECK_PORT", Value: "51515"},
655+
}
656+
637657
expected := appsv1.DaemonSet{
638658
ObjectMeta: metav1.ObjectMeta{
639659
Name: dsName,
@@ -663,7 +683,7 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() {
663683
{
664684
Args: args,
665685
Command: command,
666-
Env: env,
686+
Env: append(presetEnv, env...),
667687
Image: draImage,
668688
ImagePullPolicy: ipp,
669689
Name: "dra",
@@ -672,7 +692,6 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() {
672692
Privileged: ptr.To(true),
673693
},
674694
VolumeMounts: []v1.VolumeMount{
675-
draVolMount,
676695
{
677696
Name: "kubelet-plugins",
678697
MountPath: "/var/lib/kubelet/plugins/",
@@ -681,6 +700,23 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() {
681700
Name: "kubelet-plugins-registry",
682701
MountPath: "/var/lib/kubelet/plugins_registry/",
683702
},
703+
{
704+
Name: "cdi",
705+
MountPath: "/var/run/cdi",
706+
},
707+
draVolMount,
708+
},
709+
LivenessProbe: &v1.Probe{
710+
ProbeHandler: v1.ProbeHandler{
711+
GRPC: &v1.GRPCAction{
712+
Port: 51515,
713+
Service: ptr.To("liveness"),
714+
},
715+
},
716+
InitialDelaySeconds: 30,
717+
PeriodSeconds: 10,
718+
TimeoutSeconds: 5,
719+
FailureThreshold: 3,
684720
},
685721
},
686722
},
@@ -689,6 +725,7 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() {
689725
utils.GetKernelModuleReadyNodeLabel(namespace, draModuleName): "",
690726
},
691727
PriorityClassName: "system-node-critical",
728+
HostNetwork: true,
692729
ServiceAccountName: serviceAccountName,
693730
Volumes: []v1.Volume{
694731
{
@@ -709,6 +746,15 @@ var _ = Describe("DRAReconciler_setDRAAsDesired", func() {
709746
},
710747
},
711748
},
749+
{
750+
Name: "cdi",
751+
VolumeSource: v1.VolumeSource{
752+
HostPath: &v1.HostPathVolumeSource{
753+
Path: "/var/run/cdi",
754+
Type: &hostPathDirOrCreate,
755+
},
756+
},
757+
},
712758
draVol,
713759
},
714760
Tolerations: []v1.Toleration{testToleration},

0 commit comments

Comments
 (0)