feat(pd): initial support of multiple pd group (#6562)

Author: liubog2008

api/core/v1alpha1/cluster_types.go

@@ -86,6 +86,13 @@ type ClusterSpec struct {
 	// +listMapKey=name
 	FeatureGates []meta.FeatureGate `json:"featureGates,omitempty"`
 
+	// CustomizedPDServiceName is the internal pd service name which will be created by the cluster.
+	// If MultiPDGroup feature is not enabled, this field doesn't work.
+	// If MultiPDGroup feature is enabled, the default service name is ${clusterName}-pd.
+	// This field is useful to keep use legacy pd service name when enable MultiPDGroup.
+	CustomizedPDServiceName *string `json:"customizedPDServiceName,omitempty"`
+
+	// Security defines the security config of the whole cluster
 	Security *ClusterSecurity `json:"security,omitempty"`
 }
 
@@ -119,7 +126,14 @@ type TLSCluster struct {
 	Enabled bool `json:"enabled,omitempty"`
 }
 
+type ClusterTLSConfig struct {
+	// Client defines the client tls for the tidb operator to visit components
+	Client *InternalClientTLS `json:"client,omitempty"`
+}
+
 type ClusterSecurity struct {
+	// TLS defines tls configs for the whole cluster
+	TLS *ClusterTLSConfig `json:"tls,omitempty"`
 	// SessionTokenSigningCertKeyPair is the name of the K8s secret, where stores certificates for signing the TiDB session token,
 	// which is used by TiProxy for session migration.
 	// You can generate certificates and create a secret by: kubectl create secret generic <secret-name> --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key>

api/core/v1alpha1/common_types.go

@@ -513,13 +513,10 @@ type CertKeyPairReference struct {
 // Security defines the security config of a component
 type Security struct {
 	// TLS defines the tls configs of components
-	TLS *ComponentTLS `json:"tls,omitempty"`
+	TLS *ComponentTLSConfig `json:"tls,omitempty"`
 }
 
-// ComponentTLS defines the tls config of a component
-type ComponentTLS struct {
+// ComponentTLSConfig defines the tls config of a component
+type ComponentTLSConfig struct {
 	Cluster *InternalTLS `json:"cluster,omitempty"`
-
-	// Client defines the client tls for the tidb operator to visit the group
-	Client *InternalClientTLS `json:"client,omitempty"`
 }

api/core/v1alpha1/pd_types.go

@@ -196,10 +196,10 @@ type PDMode string
 
 const (
 	// Normal mode of PD, all services are provided by a single component
-	PDModeNormal = ""
+	PDModeNormal PDMode = ""
 	// Micro Service Mode of PD, some services(TSO,Scheduling) are separated from PD
 	// See https://docs.pingcap.com/tidb/stable/pd-microservices/
-	PDModeMS = "ms"
+	PDModeMS PDMode = "ms"
 )
 
 type PDServer struct {

api/core/v1alpha1/tidb_types.go

@@ -217,7 +217,7 @@ type TiDBPreStop struct {
 
 type TiDBSecurity struct {
 	// TLS defines the tls configs of TiDB
-	TLS *TiDBTLS `json:"tls,omitempty"`
+	TLS *TiDBTLSConfig `json:"tls,omitempty"`
 
 	// Whether enable `tidb_auth_token` authentication method.
 	// To enable this feature, a secret named `<groupName>-tidb-auth-token-jwks-secret` must be created to store the JWKs.
@@ -280,7 +280,7 @@ type TiDBSlowLog struct {
 	Resources ResourceRequirements `json:"resources,omitempty"`
 }
 
-type TiDBTLS struct {
+type TiDBTLSConfig struct {
 	// When enabled, TiDB will accept TLS encrypted connections from MySQL clients.
 	// The steps to enable this feature:
 	//   1. Generate a TiDB server-side certificate and a client-side certifiacete for the TiDB cluster.
@@ -298,8 +298,8 @@ type TiDBTLS struct {
 	// +kubebuilder:validation:XValidation:rule="oldSelf == null || self.enabled == oldSelf.enabled",message="field .mysql.enabled is immutable"
 	MySQL *TLS `json:"mysql,omitempty"`
 
-	// ComponentTLS is tls config to access internal components
-	ComponentTLS `json:",inline"`
+	// ComponentTLSConfig is tls config to access internal components
+	ComponentTLSConfig `json:",inline"`
 }
 
 type TiDBAuthToken struct {

api/core/v1alpha1/tiproxy_types.go

@@ -177,7 +177,7 @@ type TiProxyPreStop struct {
 
 type TiProxySecurity struct {
 	// Whether enable the TLS connection.
-	TLS *TiProxyTLS `json:"tls,omitempty"`
+	TLS *TiProxyTLSConfig `json:"tls,omitempty"`
 }
 
 type TiProxyServer struct {
@@ -216,7 +216,7 @@ type TiProxyProb struct {
 	Type *string `json:"type,omitempty"`
 }
 
-type TiProxyTLS struct {
+type TiProxyTLSConfig struct {
 	// MySQL defines the TLS configuration for connections between TiProxy and MySQL clients.
 	// The steps to enable this feature:
 	//   1. Generate a TiProxy server-side certificate for the TiProxy cluster.
@@ -243,9 +243,6 @@ type TiProxyTLS struct {
 	// It can be disabled by set Enabled to false.
 	// NOTE: if server tls is disabled, the Client will also be disabled.
 	Server *TLS `json:"server,omitempty"`
-
-	// Client defines the client tls for the tidb operator to visit the tiproxy group
-	Client *InternalClientTLS `json:"client,omitempty"`
 }
 
 type TiProxyGroupStatus struct {

api/core/v1alpha1/zz_generated.deepcopy.go

@@ -18,7 +18,7 @@ package v1alpha1
 // NOTE(liubo02): +enum is not supported now, we have to add all enum into comments
 // NOTE(liubo02): It's supported by https://github.com/kubernetes-sigs/controller-tools/pull/1179
 //
-// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI;SessionTokenSigning;ClusterSubdomain;TerminableLogTailer;UseTSOReadyAPI;UseSchedulingReadyAPI;UseTiKVReadyAPI;UsePDReadyAPIV2;UseTiFlashReadyAPI
+// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI;SessionTokenSigning;ClusterSubdomain;TerminableLogTailer;UseTSOReadyAPI;UseSchedulingReadyAPI;UseTiKVReadyAPI;UsePDReadyAPIV2;UseTiFlashReadyAPI;MultiPDGroup
 // +enum
 type Feature string
 
@@ -102,4 +102,12 @@ const (
 	// UseTiFlashReadyAPI means use TiFlash's /readyz API as the readiness probe.
 	UseTiFlashReadyAPI      Feature      = "UseTiFlashReadyAPI"
 	UseTiFlashReadyAPIStage FeatureStage = FeatureStageAlpha
+
+	// If this feature is enabled
+	// - More than one pd group can be created for one cluster.
+	// - A new PD service will be created for all PDGroups.
+	// - The default internal pd svc of the PDGroup will be not created.
+	// - Cannot customize advertised client port
+	MultiPDGroup      Feature      = "MultiPDGroup"
+	MultiPDGroupStage FeatureStage = FeatureStageAlpha
 )

api/meta/v1alpha1/feature.go

@@ -49,15 +49,15 @@ func GroupToSecurityTypeName(t *types.Type) string {
 // GroupToTLSTypeName returns tls type name from group type
 // TiDB and TiProxy have their own tls type
 // Others: ComponentTLS
-// TiDB and TiProxy: TiDBTLS, TiProxyTLS
+// TiDB and TiProxy: TiDBTLSConfig, TiProxyTLSConfig
 func GroupToTLSTypeName(t *types.Type) string {
 	name := strings.TrimSuffix(t.Name.Name, "Group")
 	switch name {
 	case "TiDB", "TiProxy":
-		return name + "TLS"
+		return name + "TLSConfig"
 	}
 
-	return "ComponentTLS"
+	return "ComponentTLSConfig"
 }
 
 // GroupToInternalTLSTypeName returns internal tls type name from group type

cmd/runtime-gen/generators/namer.go

@@ -298,30 +298,6 @@ func (in *$.|pub$) ClusterCASecretName() string {
 	prefix, _ := NamePrefixAndSuffix(in.GetName())
 	return prefix + "-" + in.Component() + "-cluster-secret"
 }
-
-func (in *$.|pub$) ClientCertKeyPairSecretName() string {
-	sec := in.Spec.Security
-	if sec != nil && sec.TLS != nil && sec.TLS.Client != nil && sec.TLS.Client.CertKeyPair != nil {
-		return sec.TLS.Client.CertKeyPair.Name
-	}
-	return in.Cluster() + "-cluster-client-secret"
-}
-
-func (in *$.|pub$) ClientCASecretName() string {
-	sec := in.Spec.Security
-	if sec != nil && sec.TLS != nil && sec.TLS.Client != nil && sec.TLS.Client.CA != nil {
-		return sec.TLS.Client.CA.Name
-	}
-	return in.Cluster() + "-cluster-client-secret"
-}
-
-func (in *$.|pub$) ClientInsecureSkipTLSVerify() bool {
-	sec := in.Spec.Security
-	if sec != nil && sec.TLS != nil && sec.TLS.Client != nil && sec.TLS.Client.CA != nil {
-		return sec.TLS.Client.InsecureSkipTLSVerify
-	}
-	return false
-}
 `, t)
 
 	// Generate Store interface methods for TiKV and TiFlash
@@ -556,30 +532,6 @@ func (g *$.|pub$) ClusterCASecretName() string {
 	return defaultName
 }
 
-func (g *$.|pub$) ClientCertKeyPairSecretName() string {
-	sec := g.Spec.Template.Spec.Security
-	if sec != nil && sec.TLS != nil && sec.TLS.Client != nil && sec.TLS.Client.CertKeyPair != nil {
-		return sec.TLS.Client.CertKeyPair.Name
-	}
-	return g.Cluster() + "-cluster-client-secret"
-}
-
-func (g *$.|pub$) ClientCASecretName() string {
-	sec := g.Spec.Template.Spec.Security
-	if sec != nil && sec.TLS != nil && sec.TLS.Client != nil && sec.TLS.Client.CA != nil {
-		return sec.TLS.Client.CA.Name
-	}
-	return g.Cluster() + "-cluster-client-secret"
-}
-
-func (g *$.|pub$) ClientInsecureSkipTLSVerify() bool {
-	sec := g.Spec.Template.Spec.Security
-	if sec != nil && sec.TLS != nil && sec.TLS.Client != nil && sec.TLS.Client.CA != nil {
-		return sec.TLS.Client.InsecureSkipTLSVerify
-	}
-	return false
-}
-
 func (g *$.|pub$) MinReadySeconds() int64 {
 	if g.Spec.MinReadySeconds == nil {
 		return v1alpha1.Default$.|instance$MinReadySeconds