fix: Addon job resources configuration (#10223)

(cherry picked from commit e25fa9d)

Author: leon-ape

cmd/manager/main.go

@@ -266,6 +266,12 @@ func validateRequiredToParseConfigs() error {
 			return err
 		}
 	}
+	if jobResources := viper.GetString(constant.CfgKeyAddonJobResources); jobResources != "" {
+		resources := corev1.ResourceRequirements{}
+		if err := json.Unmarshal([]byte(jobResources), &resources); err != nil {
+			return err
+		}
+	}
 	if err := validateTolerations(viper.GetString(constant.CfgKeyCtrlrMgrTolerations)); err != nil {
 		return err
 	}

controllers/extensions/addon_controller_stages.go

@@ -503,9 +503,9 @@ func setSharedVolume(addon *extensionsv1alpha1.Addon, helmJobPodSpec *corev1.Pod
 }
 
 // setInitContainer sets init containers to copy dependent charts to shared volume
-func setInitContainer(addon *extensionsv1alpha1.Addon, helmJobPodSpec *corev1.PodSpec) {
+func setInitContainer(addon *extensionsv1alpha1.Addon, helmJobPodSpec *corev1.PodSpec) error {
 	if !useLocalCharts(addon) {
-		return
+		return nil
 	}
 
 	fromPath := addon.Spec.Helm.ChartsPathInImage
@@ -515,7 +515,7 @@ func setInitContainer(addon *extensionsv1alpha1.Addon, helmJobPodSpec *corev1.Po
 	copyChartsContainer := corev1.Container{
 		Name:            "copy-charts",
 		Image:           intctrlutil.ReplaceImageRegistry(addon.Spec.Helm.ChartsImage),
-		ImagePullPolicy: corev1.PullPolicy(viper.GetString(constant.CfgAddonChartsImgPullPolicy)),
+		ImagePullPolicy: corev1.PullPolicy(viper.GetString(constant.CfgKeyAddonChartsImgPullPolicy)),
 		Command:         []string{"sh", "-c", fmt.Sprintf("cp %s/* /mnt/charts", fromPath)},
 		VolumeMounts: []corev1.VolumeMount{
 			{
@@ -524,8 +524,11 @@ func setInitContainer(addon *extensionsv1alpha1.Addon, helmJobPodSpec *corev1.Po
 			},
 		},
 	}
-	intctrlutil.InjectZeroResourcesLimitsIfEmpty(&copyChartsContainer)
+	if err := setAddonJobResourcesOrZero(&copyChartsContainer); err != nil {
+		return err
+	}
 	helmJobPodSpec.InitContainers = append(helmJobPodSpec.InitContainers, copyChartsContainer)
+	return nil
 }
 
 func (r *helmTypeInstallStage) Handle(ctx context.Context) {
@@ -687,7 +690,10 @@ func (r *helmTypeInstallStage) Handle(ctx context.Context) {
 		// we will copy the charts from charts image to shared volume. Addon container will use the
 		// charts from shared volume to install the addon.
 		setSharedVolume(addon, helmJobPodSpec)
-		setInitContainer(addon, helmJobPodSpec)
+		if err := setInitContainer(addon, helmJobPodSpec); err != nil {
+			r.setRequeueWithErr(err, "")
+			return
+		}
 
 		if err := r.reconciler.Create(ctx, helmInstallJob); err != nil {
 			r.setRequeueWithErr(err, "")
@@ -914,7 +920,7 @@ func createHelmJobProto(addon *extensionsv1alpha1.Addon) (*batchv1.Job, error) {
 	container := corev1.Container{
 		Name:            getJobMainContainerName(addon),
 		Image:           viper.GetString(constant.KBToolsImage),
-		ImagePullPolicy: corev1.PullPolicy(viper.GetString(constant.CfgAddonJobImgPullPolicy)),
+		ImagePullPolicy: corev1.PullPolicy(viper.GetString(constant.CfgKeyAddonJobImgPullPolicy)),
 		Command:         []string{"helm"},
 		Env: []corev1.EnvVar{
 			{
@@ -932,7 +938,9 @@ func createHelmJobProto(addon *extensionsv1alpha1.Addon) (*batchv1.Job, error) {
 		},
 		VolumeMounts: []corev1.VolumeMount{},
 	}
-	intctrlutil.InjectZeroResourcesLimitsIfEmpty(&container)
+	if err := setAddonJobResourcesOrZero(&container); err != nil {
+		return nil, err
+	}
 
 	helmProtoJob := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
@@ -1003,6 +1011,31 @@ func createHelmJobProto(addon *extensionsv1alpha1.Addon) (*batchv1.Job, error) {
 	return helmProtoJob, nil
 }
 
+func getAddonJobResources() (*corev1.ResourceRequirements, error) {
+	value := viper.GetString(constant.CfgKeyAddonJobResources)
+	if value == "" {
+		return nil, nil
+	}
+	resources := &corev1.ResourceRequirements{}
+	if err := json.Unmarshal([]byte(value), resources); err != nil {
+		return nil, err
+	}
+	return resources, nil
+}
+
+func setAddonJobResourcesOrZero(container *corev1.Container) error {
+	resources, err := getAddonJobResources()
+	if err != nil {
+		return err
+	}
+	if resources != nil {
+		container.Resources = *resources
+		return nil
+	}
+	intctrlutil.InjectZeroResourcesLimitsIfEmpty(container)
+	return nil
+}
+
 func enabledAddonWithDefaultValues(ctx context.Context, stageCtx *stageCtx,
 	addon *extensionsv1alpha1.Addon, reason, message string) {
 	setInstallSpec := func(di *extensionsv1alpha1.AddonDefaultInstallSpecItem) {

controllers/extensions/addon_controller_test.go

@@ -31,6 +31,7 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/version"
@@ -96,6 +97,37 @@ var _ = Describe("Addon controller", func() {
 		cleanEnv()
 	})
 
+	Context("Addon job resources", func() {
+		AfterEach(func() {
+			viper.Set(constant.CfgKeyAddonJobResources, "")
+		})
+
+		It("should apply configured resources to addon job containers", func() {
+			viper.Set(constant.CfgKeyAddonJobResources, `{"requests":{"cpu":"10m","memory":"16Mi"},"limits":{"cpu":"100m","memory":"64Mi"}}`)
+			addon := &extensionsv1alpha1.Addon{
+				ObjectMeta: metav1.ObjectMeta{Name: "test-addon"},
+				Spec: extensionsv1alpha1.AddonSpec{
+					Helm: &extensionsv1alpha1.HelmTypeInstallSpec{
+						ChartLocationURL: "file://test-chart",
+						ChartsImage:      "test-charts-image",
+					},
+				},
+			}
+
+			job, err := createHelmJobProto(addon)
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(job.Spec.Template.Spec.Containers[0].Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m")))
+			Expect(job.Spec.Template.Spec.Containers[0].Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("16Mi")))
+			Expect(job.Spec.Template.Spec.Containers[0].Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("100m")))
+			Expect(job.Spec.Template.Spec.Containers[0].Resources.Limits).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("64Mi")))
+
+			Expect(setInitContainer(addon, &job.Spec.Template.Spec)).Should(Succeed())
+			Expect(job.Spec.Template.Spec.InitContainers).Should(HaveLen(1))
+			Expect(job.Spec.Template.Spec.InitContainers[0].Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m")))
+			Expect(job.Spec.Template.Spec.InitContainers[0].Resources.Requests).Should(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("16Mi")))
+		})
+	})
+
 	Context("Addon controller test", func() {
 		var addon *extensionsv1alpha1.Addon
 		var key types.NamespacedName

deploy/helm/templates/deployment.yaml

@@ -164,6 +164,10 @@ spec:
               value: {{ .jobTTL | quote }}
             - name: ADDON_JOB_IMAGE_PULL_POLICY
               value: {{ .jobImagePullPolicy | default "IfNotPresent" }}
+            {{- with .jobResources }}
+            - name: ADDON_JOB_RESOURCES
+              value: {{ toJson . | quote }}
+            {{- end }}
             {{- end }}
             - name: KUBEBLOCKS_ADDON_HELM_INSTALL_OPTIONS
               value: {{ join " " .Values.addonHelmInstallOptions }}

deploy/helm/values.yaml

@@ -452,6 +452,9 @@ addonController:
   enabled: true
   jobTTL: "5m"
   jobImagePullPolicy: IfNotPresent
+  ## @param addonController.jobResources - resources for addon install/uninstall jobs.
+  ## Set this when namespace LimitRange policies require non-zero requests.
+  jobResources: {}
 
 
 ## @param keepAddons - keep Addon CR objects when delete this chart.

pkg/constant/viper_config.go

@@ -33,11 +33,12 @@ const (
 	CfgHostPortExcludeRanges            = "HOST_PORT_EXCLUDE_RANGES"
 
 	// addon config keys
-	CfgKeyAddonJobTTL        = "ADDON_JOB_TTL"
-	CfgAddonJobImgPullPolicy = "ADDON_JOB_IMAGE_PULL_POLICY"
+	CfgKeyAddonJobTTL           = "ADDON_JOB_TTL"
+	CfgKeyAddonJobImgPullPolicy = "ADDON_JOB_IMAGE_PULL_POLICY"
+	CfgKeyAddonJobResources     = "ADDON_JOB_RESOURCES"
 
 	// addon charts config keys
-	CfgAddonChartsImgPullPolicy = "KUBEBLOCKS_ADDON_CHARTS_IMAGE_PULL_POLICY"
+	CfgKeyAddonChartsImgPullPolicy = "KUBEBLOCKS_ADDON_CHARTS_IMAGE_PULL_POLICY"
 
 	// storage config keys
 	CfgKeyDefaultStorageClass = "DEFAULT_STORAGE_CLASS"