From 246561b1b5c51cbc1b691ca8a4b75ae8253553ff Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Tue, 7 Jul 2026 02:11:54 -0400 Subject: [PATCH] feat: add scheduling fields to EtcdCluster podTemplate.spec podTemplate.spec stops at affinity/nodeSelector/tolerations, so member pods cannot be spread across zones, given resource requests/limits, a priority class, or a non-default scheduler; with no resources set the etcd container makes every member pod BestEffort and first in line for node-pressure eviction. Add topologySpreadConstraints, resources (applied to the etcd container), priorityClassName and schedulerName to podTemplate.spec and propagate them into the StatefulSet pod template next to the existing scheduling fields. Regenerate deepcopy/CRD/api-docs. New tests assert every podTemplate scheduling field lands on the rendered StatefulSet. Signed-off-by: Xavier Lange Co-Authored-By: Claude Fable 5 Signed-off-by: Xavier Lange --- api/v1alpha1/etcdcluster_types.go | 20 ++ api/v1alpha1/zz_generated.deepcopy.go | 14 +- .../bases/operator.etcd.io_etcdclusters.yaml | 247 ++++++++++++++++++ docs/api-references/docs.md | 23 ++ internal/controller/utils.go | 6 + internal/controller/utils_test.go | 122 +++++++++ 6 files changed, 431 insertions(+), 1 deletion(-) diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index f007ae04..9a645c58 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -61,6 +61,26 @@ type PodSpec struct { Affinity *corev1.Affinity `json:"affinity,omitempty"` NodeSelector map[string]string `json:"nodeSelector,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // TopologySpreadConstraints describes how member pods should spread across topology + // domains, typically zones or hosts. The labelSelector must match the member pod + // labels, e.g. "app: ". + // +optional + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + + // Resources describes the compute resource requirements of the etcd container. + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + + // PriorityClassName is the priority class of the member pods, + // e.g. "system-cluster-critical". + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` + + // SchedulerName dispatches the member pods to a specific scheduler instead of the + // default one. + // +optional + SchedulerName string `json:"schedulerName,omitempty"` } type PodMetadata struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9db6ca12..67a33c9d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" netx "net" ) @@ -266,6 +266,18 @@ func (in *PodSpec) DeepCopyInto(out *PodSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpec. diff --git a/config/crd/bases/operator.etcd.io_etcdclusters.yaml b/config/crd/bases/operator.etcd.io_etcdclusters.yaml index e92496c1..265ed0f4 100644 --- a/config/crd/bases/operator.etcd.io_etcdclusters.yaml +++ b/config/crd/bases/operator.etcd.io_etcdclusters.yaml @@ -997,6 +997,76 @@ spec: additionalProperties: type: string type: object + priorityClassName: + description: |- + PriorityClassName is the priority class of the member pods, + e.g. "system-cluster-critical". + type: string + resources: + description: Resources describes the compute resource requirements + of the etcd container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + schedulerName: + description: |- + SchedulerName dispatches the member pods to a specific scheduler instead of the + default one. + type: string tolerations: items: description: |- @@ -1036,6 +1106,183 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how member pods should spread across topology + domains, typically zones or hosts. The labelSelector must match the member pod + labels, e.g. "app: ". + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array type: object type: object size: diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 6a32be4e..4ba7b7d8 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -153,6 +153,28 @@ _Appears in:_ | `labels` _object (keys:string, values:string)_ | | | | +#### PodSpec + + + + + + + +_Appears in:_ +- [PodTemplate](#podtemplate) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#affinity-v1-core)_ | | | | +| `nodeSelector` _object (keys:string, values:string)_ | | | | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#toleration-v1-core) array_ | | | | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#topologyspreadconstraint-v1-core) array_ | TopologySpreadConstraints describes how member pods should spread across topology
domains, typically zones or hosts. The labelSelector must match the member pod
labels, e.g. "app: ". | | Optional: \{\}
| +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#resourcerequirements-v1-core)_ | Resources describes the compute resource requirements of the etcd container. | | Optional: \{\}
| +| `priorityClassName` _string_ | PriorityClassName is the priority class of the member pods,
e.g. "system-cluster-critical". | | Optional: \{\}
| +| `schedulerName` _string_ | SchedulerName dispatches the member pods to a specific scheduler instead of the
default one. | | Optional: \{\}
| + + #### PodTemplate @@ -167,6 +189,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PodSpec](#podspec)_ | | | | #### ProviderAutoConfig diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 3092c379..98d345b5 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -222,6 +222,12 @@ func createOrPatchStatefulSet(ctx context.Context, logger logr.Logger, ec *ecv1a podSpec.Affinity = ec.Spec.PodTemplate.Spec.Affinity podSpec.NodeSelector = ec.Spec.PodTemplate.Spec.NodeSelector podSpec.Tolerations = ec.Spec.PodTemplate.Spec.Tolerations + podSpec.TopologySpreadConstraints = ec.Spec.PodTemplate.Spec.TopologySpreadConstraints + podSpec.PriorityClassName = ec.Spec.PodTemplate.Spec.PriorityClassName + podSpec.SchedulerName = ec.Spec.PodTemplate.Spec.SchedulerName + if ec.Spec.PodTemplate.Spec.Resources != nil { + podSpec.Containers[0].Resources = *ec.Spec.PodTemplate.Spec.Resources + } } // Apply custom metadata from PodTemplate if provided diff --git a/internal/controller/utils_test.go b/internal/controller/utils_test.go index 67592316..33c7bb3d 100644 --- a/internal/controller/utils_test.go +++ b/internal/controller/utils_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -701,6 +702,127 @@ func TestCreateOrPatchStatefulSetWithPodLabels(t *testing.T) { } } +func TestCreateOrPatchStatefulSetWithSchedulingFields(t *testing.T) { + ctx := t.Context() + logger := log.FromContext(ctx) + + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = ecv1alpha1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "node-role.kubernetes.io/etcd", + Operator: corev1.NodeSelectorOpExists, + }}, + }}, + }, + }, + } + nodeSelector := map[string]string{"disktype": "ssd"} + tolerations := []corev1.Toleration{{ + Key: "dedicated", + Operator: corev1.TolerationOpEqual, + Value: "etcd", + Effect: corev1.TaintEffectNoSchedule, + }} + topologySpreadConstraints := []corev1.TopologySpreadConstraint{{ + MaxSkew: 1, + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: corev1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test-etcd-scheduling"}, + }, + }} + resources := &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + ec := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-etcd-scheduling", + Namespace: "default", + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: 3, + Version: "3.5.17", + PodTemplate: &ecv1alpha1.PodTemplate{ + Spec: &ecv1alpha1.PodSpec{ + Affinity: affinity, + NodeSelector: nodeSelector, + Tolerations: tolerations, + TopologySpreadConstraints: topologySpreadConstraints, + Resources: resources, + PriorityClassName: "system-cluster-critical", + SchedulerName: "custom-scheduler", + }, + }, + }, + } + + err := createOrPatchStatefulSet(ctx, logger, ec, fakeClient, 3, scheme) + assert.NoError(t, err) + + sts := &appsv1.StatefulSet{} + err = fakeClient.Get(ctx, client.ObjectKey{Name: ec.Name, Namespace: "default"}, sts) + assert.NoError(t, err) + + podSpec := sts.Spec.Template.Spec + assert.Equal(t, affinity, podSpec.Affinity) + assert.Equal(t, nodeSelector, podSpec.NodeSelector) + assert.Equal(t, tolerations, podSpec.Tolerations) + assert.Equal(t, topologySpreadConstraints, podSpec.TopologySpreadConstraints) + assert.Equal(t, "system-cluster-critical", podSpec.PriorityClassName) + assert.Equal(t, "custom-scheduler", podSpec.SchedulerName) + assert.Equal(t, *resources, podSpec.Containers[0].Resources) +} + +func TestCreateOrPatchStatefulSetWithoutSchedulingFields(t *testing.T) { + ctx := t.Context() + logger := log.FromContext(ctx) + + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = ecv1alpha1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + ec := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-etcd-no-scheduling", + Namespace: "default", + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: 3, + Version: "3.5.17", + }, + } + + err := createOrPatchStatefulSet(ctx, logger, ec, fakeClient, 3, scheme) + assert.NoError(t, err) + + sts := &appsv1.StatefulSet{} + err = fakeClient.Get(ctx, client.ObjectKey{Name: ec.Name, Namespace: "default"}, sts) + assert.NoError(t, err) + + podSpec := sts.Spec.Template.Spec + assert.Nil(t, podSpec.TopologySpreadConstraints) + assert.Empty(t, podSpec.PriorityClassName) + assert.Empty(t, podSpec.Containers[0].Resources.Requests) + assert.Empty(t, podSpec.Containers[0].Resources.Limits) +} + func TestCreatingArgs(t *testing.T) { tests := []struct { testName string