Skip to content

Commit f2d2773

Browse files
committed
objectupdate: generelize volumes creation
we're updating the volumes for the pod and the volume mounts for the RTE container on different stages during the operator flow. This commit generelize the update flow and make sure to set all the defaults bits to avoid any unnecessary updates calls to the server. Signed-off-by: Talor Itzhak <titzhak@redhat.com>
1 parent c2ddae3 commit f2d2773

5 files changed

Lines changed: 607 additions & 51 deletions

File tree

pkg/numaresourcesscheduler/objectstate/sched/sched.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"github.com/openshift-kni/numaresources-operator/pkg/objectstate"
3636
"github.com/openshift-kni/numaresources-operator/pkg/objectstate/compare"
3737
"github.com/openshift-kni/numaresources-operator/pkg/objectstate/merge"
38+
"github.com/openshift-kni/numaresources-operator/pkg/objectupdate/volume"
3839
)
3940

4041
const (
@@ -224,14 +225,13 @@ func SchedulerNameFromObject(obj client.Object) (string, bool) {
224225
}
225226

226227
func NewSchedConfigVolume(schedVolumeConfigName, configMapName string) corev1.Volume {
227-
return corev1.Volume{
228-
Name: schedVolumeConfigName,
229-
VolumeSource: corev1.VolumeSource{
230-
ConfigMap: &corev1.ConfigMapVolumeSource{
231-
LocalObjectReference: corev1.LocalObjectReference{
232-
Name: configMapName,
233-
},
234-
},
235-
},
236-
}
228+
// Create a temporary pod spec and container to use the volume package
229+
podSpec := &corev1.PodSpec{}
230+
container := &corev1.Container{}
231+
232+
// Use the volume package to create the ConfigMap volume
233+
volume.AddConfigMap(podSpec, container, schedVolumeConfigName, "/tmp", configMapName, volume.DefaultMode, false, false)
234+
235+
// Return just the volume part (we don't need the mount)
236+
return podSpec.Volumes[0]
237237
}

pkg/objectupdate/rte/rte.go

Lines changed: 23 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121

2222
appsv1 "k8s.io/api/apps/v1"
2323
corev1 "k8s.io/api/core/v1"
24-
"k8s.io/apimachinery/pkg/api/resource"
2524
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2625
"k8s.io/apimachinery/pkg/util/intstr"
2726
"k8s.io/klog/v2"
@@ -37,6 +36,7 @@ import (
3736
nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
3837
"github.com/openshift-kni/numaresources-operator/pkg/hash"
3938
"github.com/openshift-kni/numaresources-operator/pkg/objectupdate/envvar"
39+
"github.com/openshift-kni/numaresources-operator/pkg/objectupdate/volume"
4040
)
4141

4242
// these should be provided by a deployer API
@@ -214,7 +214,9 @@ func DaemonSetArgs(ds *appsv1.DaemonSet, conf nropv1.NodeGroupConfig) error {
214214

215215
// TODO: these don't really belong here, but OTOH adding the status file without having set
216216
// the volume doesn't work either. We need a deeper refactoring in this area.
217-
AddVolumeMountMemory(podSpec, cnt, pfpStatusMountName, pfpStatusDir, 8*_MiB)
217+
if err := AddVolumeMountMemory(podSpec, cnt, pfpStatusMountName, pfpStatusDir, 8*_MiB); err != nil {
218+
return fmt.Errorf("failed to add volume mount memory: %w", err)
219+
}
218220
envvar.SetForContainer(cnt, envvar.PFPStatusDump, envvar.PFPStatusDirDefault)
219221
} else {
220222
// TODO: ditto
@@ -230,7 +232,7 @@ func DaemonSetArgs(ds *appsv1.DaemonSet, conf nropv1.NodeGroupConfig) error {
230232
func DaemonSetTolerations(ds *appsv1.DaemonSet, userTolerations []corev1.Toleration) {
231233
podSpec := &ds.Spec.Template.Spec // shortcut
232234
// cleanup undesired toleration
233-
podSpec.Tolerations = []corev1.Toleration{}
235+
podSpec.Tolerations = nil
234236
if len(userTolerations) == 0 {
235237
return
236238
}
@@ -274,50 +276,31 @@ func hasVolume(podSpec *corev1.PodSpec, volumeName string) bool {
274276
return false
275277
}
276278

277-
func AddVolumeMountMemory(podSpec *corev1.PodSpec, cnt *corev1.Container, mountName, dirName string, sizeMiB int64) {
279+
func AddVolumeMountMemory(podSpec *corev1.PodSpec, cnt *corev1.Container, mountName, dirName string, sizeMiB int64) error {
278280
// Add the requested memory volume mount
279-
cnt.VolumeMounts = append(cnt.VolumeMounts,
280-
corev1.VolumeMount{
281-
Name: mountName,
282-
MountPath: dirName,
283-
},
284-
)
285-
podSpec.Volumes = append(podSpec.Volumes,
286-
corev1.Volume{
287-
Name: mountName,
288-
VolumeSource: corev1.VolumeSource{
289-
EmptyDir: &corev1.EmptyDirVolumeSource{
290-
Medium: corev1.StorageMediumMemory,
291-
SizeLimit: resource.NewQuantity(sizeMiB, resource.BinarySI),
292-
},
293-
},
294-
},
295-
)
281+
volume.AddMemoryVolume(podSpec, cnt, mountName, dirName, sizeMiB)
296282

297283
// Add the metrics certificate volume mount only if it doesn't already exist
298284
metricsVolumeName := "rte-metrics-service-cert"
299-
if !hasVolumeMount(cnt, metricsVolumeName) {
300-
cnt.VolumeMounts = append(cnt.VolumeMounts,
301-
corev1.VolumeMount{
302-
MountPath: "/etc/secrets/rte/",
303-
Name: metricsVolumeName,
304-
ReadOnly: true,
305-
},
306-
)
285+
if !hasVolumeMount(cnt, metricsVolumeName) && !hasVolume(podSpec, metricsVolumeName) {
286+
volume.AddSecret(podSpec, cnt, metricsVolumeName, "/etc/secrets/rte/", metricsVolumeName, volume.DefaultMode, false, true)
307287
}
308288

309-
if !hasVolume(podSpec, metricsVolumeName) {
310-
podSpec.Volumes = append(podSpec.Volumes,
311-
corev1.Volume{
312-
Name: metricsVolumeName,
313-
VolumeSource: corev1.VolumeSource{
314-
Secret: &corev1.SecretVolumeSource{
315-
SecretName: metricsVolumeName,
316-
},
317-
},
318-
},
319-
)
289+
// Add host-sys volume
290+
rteSysVolumeName := "host-sys"
291+
if !hasVolume(podSpec, rteSysVolumeName) && !hasVolumeMount(cnt, rteSysVolumeName) {
292+
hostPathType := corev1.HostPathDirectory
293+
volume.AddHostPath(podSpec, cnt, rteSysVolumeName, "/host-sys", "/sys", &hostPathType, true)
294+
}
295+
296+
// Add host-podresources volume
297+
hostPodresourcesName := "host-podresources"
298+
if !hasVolume(podSpec, hostPodresourcesName) && !hasVolumeMount(cnt, hostPodresourcesName) {
299+
hostPathType := corev1.HostPathDirectory
300+
volume.AddHostPath(podSpec, cnt, hostPodresourcesName, "/host-podresources", "/var/lib/kubelet/pod-resources", &hostPathType, false)
320301
}
302+
303+
return nil
321304
}
322305

323306
func SecurityContextConstraint(scc *securityv1.SecurityContextConstraints, legacyRTEContext bool) {

pkg/objectupdate/rte/rte_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ func TestUpdateDaemonSetTolerations(t *testing.T) {
183183
{
184184
name: "defaults",
185185
conf: nropv1.NodeGroupConfig{},
186-
expectedTols: []corev1.Toleration{},
186+
expectedTols: nil,
187187
},
188188
{
189189
name: "add tolerations",

pkg/objectupdate/volume/volume.go

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*
14+
* Copyright 2025 Red Hat, Inc.
15+
*/
16+
17+
package volume
18+
19+
import (
20+
corev1 "k8s.io/api/core/v1"
21+
"k8s.io/apimachinery/pkg/api/resource"
22+
"k8s.io/klog/v2"
23+
)
24+
25+
// VolumeType represents the type of volume to create
26+
type VolumeType string
27+
28+
const (
29+
TypeConfigMap VolumeType = "configmap"
30+
TypeSecret VolumeType = "secret"
31+
TypeHostPath VolumeType = "hostpath"
32+
TypeEmptyDir VolumeType = "emptydir"
33+
)
34+
35+
const (
36+
DefaultMode = 420
37+
)
38+
39+
// Options holds the configuration for creating volumes and volume mounts
40+
type Options struct {
41+
// Common fields
42+
VolumeName string
43+
MountPath string
44+
ReadOnly bool
45+
SubPath string
46+
Type VolumeType
47+
48+
// ConfigMap/Secret specific
49+
ResourceName string
50+
DefaultMode int32
51+
Optional bool
52+
53+
// HostPath specific
54+
HostPath string
55+
HostPathType *corev1.HostPathType
56+
57+
// EmptyDir specific
58+
SizeLimit *resource.Quantity
59+
Medium corev1.StorageMedium
60+
}
61+
62+
// Add adds a volume to the pod spec and a corresponding volume mount to the container
63+
func Add(podSpec *corev1.PodSpec, container *corev1.Container, opts Options) {
64+
// Add volume mount to container
65+
volumeMount := corev1.VolumeMount{
66+
Name: opts.VolumeName,
67+
MountPath: opts.MountPath,
68+
ReadOnly: opts.ReadOnly,
69+
}
70+
if opts.SubPath != "" {
71+
volumeMount.SubPath = opts.SubPath
72+
}
73+
container.VolumeMounts = append(container.VolumeMounts, volumeMount)
74+
75+
// Create volume based on type
76+
volume := corev1.Volume{
77+
Name: opts.VolumeName,
78+
}
79+
80+
switch opts.Type {
81+
case TypeConfigMap:
82+
volume.VolumeSource = corev1.VolumeSource{
83+
ConfigMap: &corev1.ConfigMapVolumeSource{
84+
LocalObjectReference: corev1.LocalObjectReference{
85+
Name: opts.ResourceName,
86+
},
87+
Optional: &opts.Optional,
88+
DefaultMode: &opts.DefaultMode,
89+
},
90+
}
91+
case TypeSecret:
92+
volume.VolumeSource = corev1.VolumeSource{
93+
Secret: &corev1.SecretVolumeSource{
94+
SecretName: opts.ResourceName,
95+
Optional: &opts.Optional,
96+
DefaultMode: &opts.DefaultMode,
97+
},
98+
}
99+
case TypeHostPath:
100+
volume.VolumeSource = corev1.VolumeSource{
101+
HostPath: &corev1.HostPathVolumeSource{
102+
Path: opts.HostPath,
103+
Type: opts.HostPathType,
104+
},
105+
}
106+
case TypeEmptyDir:
107+
emptyDirSource := &corev1.EmptyDirVolumeSource{}
108+
if opts.Medium != "" {
109+
emptyDirSource.Medium = opts.Medium
110+
}
111+
if opts.SizeLimit != nil {
112+
emptyDirSource.SizeLimit = opts.SizeLimit
113+
}
114+
volume.VolumeSource = corev1.VolumeSource{
115+
EmptyDir: emptyDirSource,
116+
}
117+
default:
118+
klog.InfoS("unsupported volume type, skipping", "type", opts.Type)
119+
}
120+
podSpec.Volumes = append(podSpec.Volumes, volume)
121+
}
122+
123+
// AddHostPath adds a HostPath volume with the specified path and type
124+
func AddHostPath(podSpec *corev1.PodSpec, container *corev1.Container, volumeName, mountPath, hostPath string, hostPathType *corev1.HostPathType, readOnly bool) {
125+
Add(podSpec, container, Options{
126+
VolumeName: volumeName,
127+
MountPath: mountPath,
128+
ReadOnly: readOnly,
129+
Type: TypeHostPath,
130+
HostPath: hostPath,
131+
HostPathType: hostPathType,
132+
})
133+
}
134+
135+
// AddEmptyDir adds an EmptyDir volume with the specified options
136+
func AddEmptyDir(podSpec *corev1.PodSpec, container *corev1.Container, volumeName, mountPath string, medium corev1.StorageMedium, sizeLimit *resource.Quantity) {
137+
Add(podSpec, container, Options{
138+
VolumeName: volumeName,
139+
MountPath: mountPath,
140+
Type: TypeEmptyDir,
141+
Medium: medium,
142+
SizeLimit: sizeLimit,
143+
})
144+
}
145+
146+
// AddSecret adds a Secret volume with the specified options
147+
func AddSecret(podSpec *corev1.PodSpec, container *corev1.Container, volumeName, mountPath, secretName string, defaultMode int32, optional bool, readOnly bool) {
148+
Add(podSpec, container, Options{
149+
VolumeName: volumeName,
150+
MountPath: mountPath,
151+
ReadOnly: readOnly,
152+
Type: TypeSecret,
153+
ResourceName: secretName,
154+
DefaultMode: defaultMode,
155+
Optional: optional,
156+
})
157+
}
158+
159+
// AddConfigMap adds a ConfigMap volume with the specified options
160+
func AddConfigMap(podSpec *corev1.PodSpec, container *corev1.Container, volumeName, mountPath, configMapName string, defaultMode int32, optional bool, readOnly bool) {
161+
Add(podSpec, container, Options{
162+
VolumeName: volumeName,
163+
MountPath: mountPath,
164+
ReadOnly: readOnly,
165+
Type: TypeConfigMap,
166+
ResourceName: configMapName,
167+
DefaultMode: defaultMode,
168+
Optional: optional,
169+
})
170+
}
171+
172+
// AddMemoryVolume adds an EmptyDir volume with memory storage
173+
func AddMemoryVolume(podSpec *corev1.PodSpec, container *corev1.Container, volumeName, mountPath string, sizeMiB int64) {
174+
AddEmptyDir(podSpec, container, volumeName, mountPath, corev1.StorageMediumMemory, resource.NewQuantity(sizeMiB, resource.BinarySI))
175+
}

0 commit comments

Comments
 (0)