diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index f007ae04..9a645c58 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -61,6 +61,26 @@ type PodSpec struct { Affinity *corev1.Affinity `json:"affinity,omitempty"` NodeSelector map[string]string `json:"nodeSelector,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // TopologySpreadConstraints describes how member pods should spread across topology + // domains, typically zones or hosts. The labelSelector must match the member pod + // labels, e.g. "app: ". + // +optional + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + + // Resources describes the compute resource requirements of the etcd container. + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + + // PriorityClassName is the priority class of the member pods, + // e.g. "system-cluster-critical". + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` + + // SchedulerName dispatches the member pods to a specific scheduler instead of the + // default one. + // +optional + SchedulerName string `json:"schedulerName,omitempty"` } type PodMetadata struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9db6ca12..67a33c9d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" netx "net" ) @@ -266,6 +266,18 @@ func (in *PodSpec) DeepCopyInto(out *PodSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpec. diff --git a/config/crd/bases/operator.etcd.io_etcdclusters.yaml b/config/crd/bases/operator.etcd.io_etcdclusters.yaml index e92496c1..265ed0f4 100644 --- a/config/crd/bases/operator.etcd.io_etcdclusters.yaml +++ b/config/crd/bases/operator.etcd.io_etcdclusters.yaml @@ -997,6 +997,76 @@ spec: additionalProperties: type: string type: object + priorityClassName: + description: |- + PriorityClassName is the priority class of the member pods, + e.g. "system-cluster-critical". + type: string + resources: + description: Resources describes the compute resource requirements + of the etcd container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + schedulerName: + description: |- + SchedulerName dispatches the member pods to a specific scheduler instead of the + default one. + type: string tolerations: items: description: |- @@ -1036,6 +1106,183 @@ spec: type: string type: object type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how member pods should spread across topology + domains, typically zones or hosts. The labelSelector must match the member pod + labels, e.g. "app: ". + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array type: object type: object size: diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 6a32be4e..4ba7b7d8 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -153,6 +153,28 @@ _Appears in:_ | `labels` _object (keys:string, values:string)_ | | | | +#### PodSpec + + + + + + + +_Appears in:_ +- [PodTemplate](#podtemplate) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#affinity-v1-core)_ | | | | +| `nodeSelector` _object (keys:string, values:string)_ | | | | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#toleration-v1-core) array_ | | | | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#topologyspreadconstraint-v1-core) array_ | TopologySpreadConstraints describes how member pods should spread across topology
domains, typically zones or hosts. The labelSelector must match the member pod
labels, e.g. "app: ". | | Optional: \{\}
| +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#resourcerequirements-v1-core)_ | Resources describes the compute resource requirements of the etcd container. | | Optional: \{\}
| +| `priorityClassName` _string_ | PriorityClassName is the priority class of the member pods,
e.g. "system-cluster-critical". | | Optional: \{\}
| +| `schedulerName` _string_ | SchedulerName dispatches the member pods to a specific scheduler instead of the
default one. | | Optional: \{\}
| + + #### PodTemplate @@ -167,6 +189,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PodSpec](#podspec)_ | | | | #### ProviderAutoConfig diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 3092c379..98d345b5 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -222,6 +222,12 @@ func createOrPatchStatefulSet(ctx context.Context, logger logr.Logger, ec *ecv1a podSpec.Affinity = ec.Spec.PodTemplate.Spec.Affinity podSpec.NodeSelector = ec.Spec.PodTemplate.Spec.NodeSelector podSpec.Tolerations = ec.Spec.PodTemplate.Spec.Tolerations + podSpec.TopologySpreadConstraints = ec.Spec.PodTemplate.Spec.TopologySpreadConstraints + podSpec.PriorityClassName = ec.Spec.PodTemplate.Spec.PriorityClassName + podSpec.SchedulerName = ec.Spec.PodTemplate.Spec.SchedulerName + if ec.Spec.PodTemplate.Spec.Resources != nil { + podSpec.Containers[0].Resources = *ec.Spec.PodTemplate.Spec.Resources + } } // Apply custom metadata from PodTemplate if provided diff --git a/internal/controller/utils_test.go b/internal/controller/utils_test.go index 67592316..33c7bb3d 100644 --- a/internal/controller/utils_test.go +++ b/internal/controller/utils_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -701,6 +702,127 @@ func TestCreateOrPatchStatefulSetWithPodLabels(t *testing.T) { } } +func TestCreateOrPatchStatefulSetWithSchedulingFields(t *testing.T) { + ctx := t.Context() + logger := log.FromContext(ctx) + + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = ecv1alpha1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "node-role.kubernetes.io/etcd", + Operator: corev1.NodeSelectorOpExists, + }}, + }}, + }, + }, + } + nodeSelector := map[string]string{"disktype": "ssd"} + tolerations := []corev1.Toleration{{ + Key: "dedicated", + Operator: corev1.TolerationOpEqual, + Value: "etcd", + Effect: corev1.TaintEffectNoSchedule, + }} + topologySpreadConstraints := []corev1.TopologySpreadConstraint{{ + MaxSkew: 1, + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: corev1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test-etcd-scheduling"}, + }, + }} + resources := &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + ec := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-etcd-scheduling", + Namespace: "default", + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: 3, + Version: "3.5.17", + PodTemplate: &ecv1alpha1.PodTemplate{ + Spec: &ecv1alpha1.PodSpec{ + Affinity: affinity, + NodeSelector: nodeSelector, + Tolerations: tolerations, + TopologySpreadConstraints: topologySpreadConstraints, + Resources: resources, + PriorityClassName: "system-cluster-critical", + SchedulerName: "custom-scheduler", + }, + }, + }, + } + + err := createOrPatchStatefulSet(ctx, logger, ec, fakeClient, 3, scheme) + assert.NoError(t, err) + + sts := &appsv1.StatefulSet{} + err = fakeClient.Get(ctx, client.ObjectKey{Name: ec.Name, Namespace: "default"}, sts) + assert.NoError(t, err) + + podSpec := sts.Spec.Template.Spec + assert.Equal(t, affinity, podSpec.Affinity) + assert.Equal(t, nodeSelector, podSpec.NodeSelector) + assert.Equal(t, tolerations, podSpec.Tolerations) + assert.Equal(t, topologySpreadConstraints, podSpec.TopologySpreadConstraints) + assert.Equal(t, "system-cluster-critical", podSpec.PriorityClassName) + assert.Equal(t, "custom-scheduler", podSpec.SchedulerName) + assert.Equal(t, *resources, podSpec.Containers[0].Resources) +} + +func TestCreateOrPatchStatefulSetWithoutSchedulingFields(t *testing.T) { + ctx := t.Context() + logger := log.FromContext(ctx) + + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = ecv1alpha1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + ec := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-etcd-no-scheduling", + Namespace: "default", + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: 3, + Version: "3.5.17", + }, + } + + err := createOrPatchStatefulSet(ctx, logger, ec, fakeClient, 3, scheme) + assert.NoError(t, err) + + sts := &appsv1.StatefulSet{} + err = fakeClient.Get(ctx, client.ObjectKey{Name: ec.Name, Namespace: "default"}, sts) + assert.NoError(t, err) + + podSpec := sts.Spec.Template.Spec + assert.Nil(t, podSpec.TopologySpreadConstraints) + assert.Empty(t, podSpec.PriorityClassName) + assert.Empty(t, podSpec.Containers[0].Resources.Requests) + assert.Empty(t, podSpec.Containers[0].Resources.Limits) +} + func TestCreatingArgs(t *testing.T) { tests := []struct { testName string