Skip to content

Commit ce69be2

Browse files
leon-apeleon-ape
authored
chore: support cluster default resources (#10227)
1 parent a4a0b88 commit ce69be2

7 files changed

Lines changed: 235 additions & 17 deletions

File tree

‎cmd/manager/main.go‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ func init() {
152152
viper.SetDefault(constant.CfgHostPortConfigMapName, "kubeblocks-host-ports")
153153
viper.SetDefault(constant.CfgHostPortIncludeRanges, "55000-59999")
154154
viper.SetDefault(constant.CfgHostPortExcludeRanges, "6443,10250,10257,10259,2379-2380,30000-32767")
155+
viper.SetDefault(constant.CfgKeyClusterDefaultResources, `{"zero":true}`)
155156
viper.SetDefault(constant.KubernetesClusterDomainEnv, constant.DefaultDNSDomain)
156157
viper.SetDefault(instanceset.MaxPlainRevisionCount, 1024)
157158
viper.SetDefault(instanceset.FeatureGateIgnorePodVerticalScaling, false)
@@ -284,7 +285,16 @@ func validateRequiredToParseConfigs() error {
284285
return err
285286
}
286287
}
287-
288+
if clusterDefaultResources := viper.GetString(constant.CfgKeyClusterDefaultResources); clusterDefaultResources != "" {
289+
resources := struct {
290+
Zero bool `json:"zero,omitempty"`
291+
Requests corev1.ResourceList `json:"requests,omitempty"`
292+
Limits corev1.ResourceList `json:"limits,omitempty"`
293+
}{}
294+
if err := json.Unmarshal([]byte(clusterDefaultResources), &resources); err != nil {
295+
return err
296+
}
297+
}
288298
if imagePullSecrets := viper.GetString(constant.KBImagePullSecrets); imagePullSecrets != "" {
289299
secrets := make([]corev1.LocalObjectReference, 0)
290300
if err := json.Unmarshal([]byte(imagePullSecrets), &secrets); err != nil {

‎deploy/helm/templates/deployment.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,8 @@ spec:
145145
value: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.tools.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
146146
- name: KUBEBLOCKS_SERVICEACCOUNT_NAME
147147
value: {{ include "kubeblocks.serviceAccountName" . }}
148+
- name: CLUSTER_DEFAULT_RESOURCES
149+
value: {{ .Values.clusterDefaultResources | toJson | quote }}
148150
{{- if .Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1" }}
149151
- name: VOLUMESNAPSHOT_API_BETA
150152
value: "false"

‎deploy/helm/values.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,19 @@ resources:
257257
cpu: 500m
258258
memory: 1Gi
259259

260+
## Default resources for init and sidecar containers in KubeBlocks-managed Cluster workloads.
261+
## Main containers are not affected; they are controlled by Cluster/Component resources.
262+
## If a resource name is specified in requests or limits, both request and limit are set for that resource name.
263+
## For unspecified resource names, zero controls whether to keep the legacy zero-limit behavior.
264+
##
265+
## @param clusterDefaultResources.zero
266+
## @param clusterDefaultResources.requests
267+
## @param clusterDefaultResources.limits
268+
clusterDefaultResources:
269+
zero: true
270+
requests: {}
271+
limits: {}
272+
260273
## @param priorityClassName
261274
##
262275
priorityClassName:

‎pkg/constant/viper_config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ const (
3131
CfgHostPortConfigMapName = "HOST_PORT_CM_NAME"
3232
CfgHostPortIncludeRanges = "HOST_PORT_INCLUDE_RANGES"
3333
CfgHostPortExcludeRanges = "HOST_PORT_EXCLUDE_RANGES"
34+
CfgKeyClusterDefaultResources = "CLUSTER_DEFAULT_RESOURCES"
3435

3536
// addon config keys
3637
CfgKeyAddonJobTTL = "ADDON_JOB_TTL"

‎pkg/controller/factory/builder.go‎

Lines changed: 87 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,9 @@ func BuildInstanceSet(synthesizedComp *component.SynthesizedComponent, compDef *
100100

101101
itsObj := itsBuilder.GetObject()
102102

103-
setDefaultResourceLimits(itsObj)
103+
if err := setDefaultResourceLimits(itsObj); err != nil {
104+
return nil, err
105+
}
104106

105107
return itsObj, nil
106108
}
@@ -237,14 +239,94 @@ func getBuiltinContainer(synthesizedComp *component.SynthesizedComponent, contai
237239
return nil
238240
}
239241

240-
func setDefaultResourceLimits(its *workloads.InstanceSet) {
241-
for _, cc := range []*[]corev1.Container{&its.Spec.Template.Spec.Containers, &its.Spec.Template.Spec.InitContainers} {
242-
for i := range *cc {
243-
intctrlutil.InjectZeroResourcesLimitsIfEmpty(&(*cc)[i])
242+
func setDefaultResourceLimits(its *workloads.InstanceSet) error {
243+
clusterResources, err := getClusterDefaultResources()
244+
if err != nil {
245+
return err
246+
}
247+
for i := range its.Spec.Template.Spec.Containers {
248+
container := &its.Spec.Template.Spec.Containers[i]
249+
if i > 0 {
250+
setClusterDefaultResources(container, clusterResources)
251+
continue
252+
}
253+
intctrlutil.InjectZeroResourcesLimitsIfEmpty(container)
254+
}
255+
for i := range its.Spec.Template.Spec.InitContainers {
256+
setClusterDefaultResources(&its.Spec.Template.Spec.InitContainers[i], clusterResources)
257+
}
258+
return nil
259+
}
260+
261+
type clusterDefaultResources struct {
262+
Zero bool `json:"zero,omitempty"`
263+
Requests corev1.ResourceList `json:"requests,omitempty"`
264+
Limits corev1.ResourceList `json:"limits,omitempty"`
265+
}
266+
267+
func getClusterDefaultResources() (clusterDefaultResources, error) {
268+
resources := clusterDefaultResources{}
269+
value := viper.GetString(constant.CfgKeyClusterDefaultResources)
270+
if value == "" {
271+
return resources, nil
272+
}
273+
if err := json.Unmarshal([]byte(value), &resources); err != nil {
274+
return clusterDefaultResources{}, err
275+
}
276+
return resources, nil
277+
}
278+
279+
func setClusterDefaultResources(container *corev1.Container, resources clusterDefaultResources) {
280+
for _, name := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} {
281+
if hasClusterDefaultResource(resources, name) {
282+
completeResource(container, resources, name)
283+
continue
284+
}
285+
if resources.Zero {
286+
intctrlutil.InjectZeroResourceLimitIfEmpty(container, name)
244287
}
245288
}
246289
}
247290

291+
func hasClusterDefaultResource(resources clusterDefaultResources, name corev1.ResourceName) bool {
292+
_, hasRequest := resources.Requests[name]
293+
_, hasLimit := resources.Limits[name]
294+
return hasRequest || hasLimit
295+
}
296+
297+
func completeResource(container *corev1.Container, resources clusterDefaultResources, name corev1.ResourceName) {
298+
if container.Resources.Requests == nil {
299+
container.Resources.Requests = corev1.ResourceList{}
300+
}
301+
if container.Resources.Limits == nil {
302+
container.Resources.Limits = corev1.ResourceList{}
303+
}
304+
305+
request, hasRequest := container.Resources.Requests[name]
306+
limit, hasLimit := container.Resources.Limits[name]
307+
if hasRequest && hasLimit {
308+
return
309+
}
310+
if hasRequest {
311+
container.Resources.Limits[name] = request
312+
return
313+
}
314+
if hasLimit {
315+
container.Resources.Requests[name] = limit
316+
return
317+
}
318+
319+
request, hasRequest = resources.Requests[name]
320+
limit, hasLimit = resources.Limits[name]
321+
if hasRequest && !hasLimit {
322+
limit = request
323+
} else if !hasRequest && hasLimit {
324+
request = limit
325+
}
326+
container.Resources.Requests[name] = request
327+
container.Resources.Limits[name] = limit
328+
}
329+
248330
// BuildPersistentVolumeClaimLabels builds a pvc name label, and synchronize the labels from component to pvc.
249331
func BuildPersistentVolumeClaimLabels(component *component.SynthesizedComponent, pvc *corev1.PersistentVolumeClaim,
250332
pvcTplName, templateName string) {

‎pkg/controller/factory/builder_test.go‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727

2828
corev1 "k8s.io/api/core/v1"
2929
rbacv1 "k8s.io/api/rbac/v1"
30+
"k8s.io/apimachinery/pkg/api/resource"
3031
"k8s.io/apimachinery/pkg/types"
3132
"sigs.k8s.io/controller-runtime/pkg/client"
3233

@@ -195,4 +196,112 @@ var _ = Describe("builder", func() {
195196
Expect(rb.Name).Should(Equal(expectName))
196197
})
197198
})
199+
200+
Context("cluster default resources", func() {
201+
AfterEach(func() {
202+
viper.Set(constant.CfgKeyClusterDefaultResources, "")
203+
})
204+
205+
newInstanceSet := func() *workloads.InstanceSet {
206+
return &workloads.InstanceSet{
207+
Spec: workloads.InstanceSetSpec{
208+
Template: corev1.PodTemplateSpec{
209+
Spec: corev1.PodSpec{
210+
Containers: []corev1.Container{
211+
{Name: "main"},
212+
{Name: "sidecar"},
213+
},
214+
InitContainers: []corev1.Container{
215+
{Name: "init"},
216+
},
217+
},
218+
},
219+
},
220+
}
221+
}
222+
223+
It("does not inject zero resources into init and sidecar containers when not configured", func() {
224+
its := newInstanceSet()
225+
226+
Expect(setDefaultResourceLimits(its)).Should(Succeed())
227+
228+
Expect(its.Spec.Template.Spec.Containers[0].Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("0")))
229+
Expect(its.Spec.Template.Spec.Containers[1].Resources.Requests).Should(BeNil())
230+
Expect(its.Spec.Template.Spec.Containers[1].Resources.Limits).Should(BeNil())
231+
Expect(its.Spec.Template.Spec.InitContainers[0].Resources.Requests).Should(BeNil())
232+
Expect(its.Spec.Template.Spec.InitContainers[0].Resources.Limits).Should(BeNil())
233+
})
234+
235+
It("keeps zero resource limit behavior when zero is true", func() {
236+
viper.Set(constant.CfgKeyClusterDefaultResources, `{"zero":true,"requests":{},"limits":{}}`)
237+
its := newInstanceSet()
238+
239+
Expect(setDefaultResourceLimits(its)).Should(Succeed())
240+
241+
Expect(its.Spec.Template.Spec.Containers[0].Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("0")))
242+
Expect(its.Spec.Template.Spec.Containers[1].Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("0")))
243+
Expect(its.Spec.Template.Spec.InitContainers[0].Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("0")))
244+
})
245+
246+
It("applies configured resources to init and sidecar containers", func() {
247+
viper.Set(constant.CfgKeyClusterDefaultResources, `{"zero":true,"requests":{"cpu":"10m","memory":"16Mi"},"limits":{"cpu":"100m","memory":"64Mi"}}`)
248+
its := newInstanceSet()
249+
250+
Expect(setDefaultResourceLimits(its)).Should(Succeed())
251+
252+
main := its.Spec.Template.Spec.Containers[0]
253+
sidecar := its.Spec.Template.Spec.Containers[1]
254+
initContainer := its.Spec.Template.Spec.InitContainers[0]
255+
Expect(main.Resources.Requests).Should(BeNil())
256+
Expect(main.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("0")))
257+
Expect(sidecar.Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m")))
258+
Expect(sidecar.Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("16Mi")))
259+
Expect(sidecar.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("100m")))
260+
Expect(sidecar.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("64Mi")))
261+
Expect(initContainer.Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m")))
262+
Expect(initContainer.Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("16Mi")))
263+
Expect(initContainer.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("100m")))
264+
Expect(initContainer.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("64Mi")))
265+
})
266+
267+
It("lets configured resource names override zero by resource name", func() {
268+
viper.Set(constant.CfgKeyClusterDefaultResources, `{"zero":true,"requests":{"cpu":"10m"},"limits":{}}`)
269+
its := newInstanceSet()
270+
271+
Expect(setDefaultResourceLimits(its)).Should(Succeed())
272+
273+
sidecar := its.Spec.Template.Spec.Containers[1]
274+
initContainer := its.Spec.Template.Spec.InitContainers[0]
275+
Expect(sidecar.Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m")))
276+
Expect(sidecar.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m")))
277+
Expect(sidecar.Resources.Requests).ShouldNot(HaveKey(corev1.ResourceMemory))
278+
Expect(sidecar.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("0")))
279+
Expect(initContainer.Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m")))
280+
Expect(initContainer.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m")))
281+
Expect(initContainer.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("0")))
282+
})
283+
284+
It("does not override sidecar resource values already set by definitions", func() {
285+
viper.Set(constant.CfgKeyClusterDefaultResources, `{"zero":true,"requests":{"cpu":"10m","memory":"16Mi"},"limits":{"cpu":"100m","memory":"64Mi"}}`)
286+
its := newInstanceSet()
287+
its.Spec.Template.Spec.Containers[1].Resources.Requests = corev1.ResourceList{
288+
corev1.ResourceCPU: resource.MustParse("250m"),
289+
}
290+
291+
Expect(setDefaultResourceLimits(its)).Should(Succeed())
292+
293+
sidecar := its.Spec.Template.Spec.Containers[1]
294+
Expect(sidecar.Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("250m")))
295+
Expect(sidecar.Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("16Mi")))
296+
Expect(sidecar.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("250m")))
297+
Expect(sidecar.Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("64Mi")))
298+
})
299+
300+
It("returns an error when cluster default resources are invalid", func() {
301+
viper.Set(constant.CfgKeyClusterDefaultResources, `{"zero":`)
302+
its := newInstanceSet()
303+
304+
Expect(setDefaultResourceLimits(its)).ShouldNot(Succeed())
305+
})
306+
})
198307
})

‎pkg/controllerutil/container_util.go‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,19 @@ func GetContainerByName(containers []corev1.Container, name string) (int, *corev
3434
}
3535

3636
func InjectZeroResourcesLimitsIfEmpty(c *corev1.Container) {
37-
zeroValue := resource.MustParse("0")
37+
InjectZeroResourceLimitIfEmpty(c, corev1.ResourceCPU)
38+
InjectZeroResourceLimitIfEmpty(c, corev1.ResourceMemory)
39+
}
40+
41+
func InjectZeroResourceLimitIfEmpty(c *corev1.Container, name corev1.ResourceName) {
42+
if _, ok := c.Resources.Requests[name]; ok {
43+
return
44+
}
45+
if _, ok := c.Resources.Limits[name]; ok {
46+
return
47+
}
3848
if c.Resources.Limits == nil {
3949
c.Resources.Limits = corev1.ResourceList{}
4050
}
41-
42-
safeSetLimitValue := func(name corev1.ResourceName) {
43-
if _, ok := c.Resources.Requests[name]; !ok {
44-
if _, ok = c.Resources.Limits[name]; !ok {
45-
c.Resources.Limits[name] = zeroValue
46-
}
47-
}
48-
}
49-
safeSetLimitValue(corev1.ResourceCPU)
50-
safeSetLimitValue(corev1.ResourceMemory)
51+
c.Resources.Limits[name] = resource.MustParse("0")
5152
}

0 commit comments

Comments
 (0)