Merge branch 'feature/build-pod-resource' into 'main'

Teach runner how to set pod-level resources for build pods

Closes #39085

See merge request https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/5922

Merged-by: Romuald Atchadé <ratchade@gitlab.com>
Approved-by: Evan Read <eread@gitlab.com>
Approved-by: Sam Roque-Worcel <sroque-worcel@gitlab.com>
Approved-by: Romuald Atchadé <ratchade@gitlab.com>
Approved-by: Roshni Sarangadharan <rsarangadharan@gitlab.com>
Reviewed-by: Stéphane Talbot <stephane.talbot@univ-savoie.fr>
Reviewed-by: Alper Akgun <aakgun@gitlab.com>
Reviewed-by: Romuald Atchadé <ratchade@gitlab.com>
Reviewed-by: GitLab Duo <gitlab-duo@gitlab.com>
Reviewed-by: Sam Roque-Worcel <sroque-worcel@gitlab.com>
Co-authored-by: Stephane Talbot <Stephane.Talbot@univ-savoie.fr>
Co-authored-by: Stéphane Talbot <stephane.talbot@univ-savoie.fr>

Author: 2 people

common/config.go

@@ -669,6 +669,14 @@ type KubernetesConfig struct {
 	HelperEphemeralStorageLimitOverwriteMaxAllowed    string                             `toml:"helper_ephemeral_storage_limit_overwrite_max_allowed,omitempty" json:"helper_ephemeral_storage_limit_overwrite_max_allowed" long:"helper-ephemeral_storage-limit-overwrite-max-allowed" env:"KUBERNETES_HELPER_EPHEMERAL_STORAGE_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the helper ephemeral storage limit can be set to. Used with the KUBERNETES_HELPER_EPHEMERAL_STORAGE_LIMIT variable in the build."`
 	HelperEphemeralStorageRequest                     string                             `toml:"helper_ephemeral_storage_request,omitempty" json:"helper_ephemeral_storage_request" long:"helper-ephemeral_storage-request" env:"KUBERNETES_HELPER_EPHEMERAL_STORAGE_REQUEST" description:"The amount of ephemeral storage requested for build helper containers"`
 	HelperEphemeralStorageRequestOverwriteMaxAllowed  string                             `toml:"helper_ephemeral_storage_request_overwrite_max_allowed,omitempty" json:"helper_ephemeral_storage_request_overwrite_max_allowed" long:"helper-ephemeral_storage-request-overwrite-max-allowed" env:"KUBERNETES_HELPER_EPHEMERAL_STORAGE_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the helper ephemeral storage request can be set to. Used with the KUBERNETES_HELPER_EPHEMERAL_STORAGE_REQUEST variable in the build."`
+	PodCPULimit                                       string                             `toml:"pod_cpu_limit,omitempty" json:"pod_cpu_limit" long:"pod-cpu-limit" env:"KUBERNETES_POD_CPU_LIMIT" description:"The CPU allocation given to the build pod"`
+	PodCPULimitOverwriteMaxAllowed                    string                             `toml:"pod_cpu_limit_overwrite_max_allowed,omitempty" json:"pod_cpu_limit_overwrite_max_allowed" long:"pod-cpu-limit-overwrite-max-allowed" env:"KUBERNETES_POD_CPU_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the maximum amount the pod CPU limit can be set to. Used with the KUBERNETES_POD_CPU_LIMIT variable in the build."`
+	PodCPURequest                                     string                             `toml:"pod_cpu_request,omitempty" json:"pod_cpu_request" long:"pod-cpu-request" env:"KUBERNETES_POD_CPU_REQUEST" description:"The CPU allocation requested for the build pod"`
+	PodCPURequestOverwriteMaxAllowed                  string                             `toml:"pod_cpu_request_overwrite_max_allowed,omitempty" json:"pod_cpu_request_overwrite_max_allowed" long:"pod-cpu-request-overwrite-max-allowed" env:"KUBERNETES_POD_CPU_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the maximum amount the pod CPU request can be set to. Used with the KUBERNETES_POD_CPU_REQUEST variable in the build."`
+	PodMemoryLimit                                    string                             `toml:"pod_memory_limit,omitempty" json:"pod_memory_limit" long:"pod-memory-limit" env:"KUBERNETES_POD_MEMORY_LIMIT" description:"The amount of memory allocated to the build pod"`
+	PodMemoryLimitOverwriteMaxAllowed                 string                             `toml:"pod_memory_limit_overwrite_max_allowed,omitempty" json:"pod_memory_limit_overwrite_max_allowed" long:"pod-memory-limit-overwrite-max-allowed" env:"KUBERNETES_POD_MEMORY_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the maximum amount the pod memory limit can be set to. Used with the KUBERNETES_POD_MEMORY_LIMIT variable in the build."`
+	PodMemoryRequest                                  string                             `toml:"pod_memory_request,omitempty" json:"pod_memory_request" long:"pod-memory-request" env:"KUBERNETES_POD_MEMORY_REQUEST" description:"The amount of memory requested from the build pod"`
+	PodMemoryRequestOverwriteMaxAllowed               string                             `toml:"pod_memory_request_overwrite_max_allowed,omitempty" json:"pod_memory_request_overwrite_max_allowed" long:"pod-memory-request-overwrite-max-allowed" env:"KUBERNETES_POD_MEMORY_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the maximum amount the pod memory request can be set to. Used with the KUBERNETES_POD_MEMORY_REQUEST variable in the build."`
 	AllowedImages                                     []string                           `toml:"allowed_images,omitempty" json:"allowed_images,omitempty" long:"allowed-images" env:"KUBERNETES_ALLOWED_IMAGES" description:"Image allowlist"`
 	AllowedPullPolicies                               []DockerPullPolicy                 `toml:"allowed_pull_policies,omitempty" json:"allowed_pull_policies,omitempty" long:"allowed-pull-policies" env:"KUBERNETES_ALLOWED_PULL_POLICIES" description:"Pull policy allowlist"`
 	AllowedServices                                   []string                           `toml:"allowed_services,omitempty" json:"allowed_services,omitempty" long:"allowed-services" env:"KUBERNETES_ALLOWED_SERVICES" description:"Service allowlist"`

docs/executors/kubernetes/_index.md

@@ -234,6 +234,16 @@ Use the following settings in the `config.toml` file to configure the Kubernetes
 | `service_cpu_limit_overwrite_max_allowed`   | The maximum amount that the CPU allocation can be written to for service containers. When empty, it disables the CPU limit overwrite feature. |
 | `service_cpu_request`                       | The CPU allocation requested for build service containers. |
 | `service_cpu_request_overwrite_max_allowed` | The maximum amount that the CPU allocation request can be written to for service containers. When empty, it disables the CPU request overwrite feature. |
+| `pod_cpu_limit`                             | The CPU allocation given to build pod. |
+| `pod_cpu_limit_overwrite_max_allowed`       | The maximum amount that the CPU allocation can be written to for build pod. When empty, it disables the CPU limit overwrite feature. |
+| `pod_cpu_request`                           | The CPU allocation requested for build pod. |
+| `pod_cpu_request_overwrite_max_allowed`     | The maximum amount that the CPU allocation request can be written to for build pod. When empty, it disables the CPU request overwrite feature. |
+
+{{< alert type="note" >}}
+
+Pod-level resource specifications have been introduced as alpha features in [Kubernetes v1.32](https://v1-32.docs.kubernetes.io/blog/2024/12/11/kubernetes-v1-32-release/#pod-level-resource-specifications) and graduated to beta in [Kubernetes v1.34](https://kubernetes.io/blog/2025/09/22/kubernetes-v1-34-pod-level-resources/).
+
+{{< /alert >}}
 
 ### Memory requests and limits
 
@@ -251,6 +261,10 @@ Use the following settings in the `config.toml` file to configure the Kubernetes
 | `service_memory_limit_overwrite_max_allowed`   | The maximum amount that the memory allocation can be written to for service containers. When empty, it disables the memory limit overwrite feature. |
 | `service_memory_request`                       | The amount of memory requested for build service containers. |
 | `service_memory_request_overwrite_max_allowed` | The maximum amount that the memory allocation request can be written to for service containers. When empty, it disables the memory request overwrite feature. |
+| `pod_memory_limit`                             | The amount of memory allocated to build pod. |
+| `pod_memory_limit_overwrite_max_allowed`       | The maximum amount that the memory allocation can be written to for build pod. When empty, it disables the memory limit overwrite feature. |
+| `pod_memory_request`                           | The amount of memory requested for build pod. |
+| `pod_memory_request_overwrite_max_allowed`     | The maximum amount that the memory allocation request can be written to for build pod. When empty, it disables the memory request overwrite feature. |
 
 #### Helper container memory sizing recommendations
 

executors/kubernetes/internal/watchers/pod_test.go

@@ -187,7 +187,7 @@ func TestPodWatcherWrongObject(t *testing.T) {
 			handler := podWatcher.resourceHandler()
 
 			assert.NotPanics(t, func() {
-				handler.OnAdd(test.object, true)
+				handler.OnAdd(test.object, false)
 			})
 		})
 	}

executors/kubernetes/kubernetes.go

@@ -1140,6 +1140,17 @@ func (s *executor) initContainerResources() api.ResourceRequirements {
 	return resources
 }
 
+func (s *executor) podResourcesReference() *api.ResourceRequirements {
+	resources := api.ResourceRequirements{}
+
+	if s.configurationOverwrites != nil {
+		resources.Limits = s.configurationOverwrites.podLimits
+		resources.Requests = s.configurationOverwrites.podRequests
+	}
+
+	return &resources
+}
+
 func (s *executor) buildPermissionsInitContainer(os string) (api.Container, error) {
 	pullPolicy, err := s.pullManager.GetPullPolicyFor(helperContainerName)
 	if err != nil {
@@ -2368,6 +2379,7 @@ func (s *executor) preparePodConfig(opts podConfigPrepareOpts) (api.Pod, error)
 			DNSConfig:                     s.Config.Kubernetes.GetDNSConfig(),
 			RuntimeClassName:              s.Config.Kubernetes.RuntimeClassName,
 			PriorityClassName:             s.Config.Kubernetes.PriorityClassName,
+			Resources:                     s.podResourcesReference(),
 		},
 	}
 

executors/kubernetes/kubernetes_integration_test.go

@@ -33,8 +33,10 @@ import (
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	versionutil "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/watch"
 	k8s "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -2883,6 +2885,130 @@ func runMultiPullPolicyBuild(t *testing.T, build *common.Build) error {
 	return err
 }
 
+func mustCreateResourceList(t *testing.T, cpu, memory string) v1.ResourceList {
+	var rCPU, rMemory resource.Quantity
+	var err error
+	if cpu != "" {
+		rCPU, err = resource.ParseQuantity(cpu)
+	}
+	require.NoError(t, err)
+
+	if memory != "" {
+		rMemory, err = resource.ParseQuantity(memory)
+	}
+	require.NoError(t, err)
+
+	resources := make(v1.ResourceList)
+	q := resource.Quantity{}
+
+	if rCPU != q {
+		resources[v1.ResourceCPU] = rCPU
+	}
+	if rMemory != q {
+		resources[v1.ResourceMemory] = rMemory
+	}
+
+	return resources
+}
+
+func skipKubectlIntegrationTestsIfNotOnLinux(t *testing.T, client *k8s.Clientset) {
+	nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	require.NoError(t, err)
+
+	os := nodes.Items[0].Status.NodeInfo.OperatingSystem
+
+	// skip tests on windows cluster
+	if os != "linux" {
+		t.Skip("Non linux -- skipping tests")
+	}
+}
+
+func skipKubectlIntegrationTestsIfOnOldCluster(t *testing.T, client *k8s.Clientset, minimalVersion string) {
+	serverVersion, err := client.Discovery().ServerVersion()
+	require.NoError(t, err)
+
+	version, err := versionutil.Parse(serverVersion.String())
+	require.NoError(t, err)
+
+	res, err := version.Compare(minimalVersion)
+	require.NoError(t, err)
+
+	// skip tests if cluster is below minimalVersion
+	if res == -1 {
+		t.Skipf("Kubernetes server (%s) is older than %s -- skipping tests", serverVersion.String(), minimalVersion)
+	}
+}
+
+func TestKubernetesBuildPodResources(t *testing.T) {
+	t.Parallel()
+
+	kubernetes.SkipKubectlIntegrationTests(t, "kubectl", "cluster-info")
+
+	client := getTestKubeClusterClient(t)
+
+	// Pod Level Resources Graduated to Beta in kubernetes v1.34
+	skipKubectlIntegrationTestsIfOnOldCluster(t, client, "1.34.0")
+	// Pod-level resources are not supported for Windows pods
+	skipKubectlIntegrationTestsIfNotOnLinux(t, client)
+
+	ctxTimeout := time.Minute
+
+	tests := map[string]struct {
+		resources map[string]string
+		verifyFn  func(*testing.T, v1.Pod)
+	}{
+		"set all pod-level resources": {
+			resources: map[string]string{
+				"PodCPURequest":    "1",
+				"PodCPULimit":      "4",
+				"PodMemoryRequest": "1Gi",
+				"PodMemoryLimit":   "8Gi",
+			},
+			verifyFn: func(t *testing.T, pod v1.Pod) {
+				resources := pod.Spec.Resources
+				expectedRequests := mustCreateResourceList(t, "1", "1Gi")
+				expectedLimits := mustCreateResourceList(t, "4", "8Gi")
+
+				require.NotNil(t, resources)
+				assert.Equal(t, expectedRequests, resources.Requests)
+				assert.Equal(t, expectedLimits, resources.Limits)
+			},
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			build := getTestBuild(t, common.GetRemoteSuccessfulBuild)
+
+			build.Runner.Kubernetes.PodCPURequest = test.resources["PodCPURequest"]
+			build.Runner.Kubernetes.PodCPULimit = test.resources["PodCPULimit"]
+			build.Runner.Kubernetes.PodMemoryRequest = test.resources["PodMemoryRequest"]
+			build.Runner.Kubernetes.PodMemoryLimit = test.resources["PodMemoryLimit"]
+
+			defer buildtest.OnUserStage(build, func() {
+				ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout)
+				defer cancel()
+				pods, err := client.CoreV1().Pods(ciNamespace).List(
+					ctx,
+					metav1.ListOptions{
+						LabelSelector: labels.Set(build.Runner.Kubernetes.PodLabels).String(),
+					},
+				)
+				require.NoError(t, err)
+				require.NotEmpty(t, pods.Items)
+				pod := pods.Items[0]
+
+				test.verifyFn(t, pod)
+			})()
+
+			err := build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
+			assert.NoError(t, err)
+		})
+	}
+}
+
 func TestKubernetesAllowedImages(t *testing.T) {
 	t.Parallel()