feat(pd): support ready api v2 (#6522)

liubog2008 · web-flow · commit 6fdada64a959 · 2025-10-28T06:23:46.000Z
diff --git a/api/meta/v1alpha1/feature.go b/api/meta/v1alpha1/feature.go
@@ -18,7 +18,7 @@ package v1alpha1
 // NOTE(liubo02): +enum is not supported now, we have to add all enum into comments
 // NOTE(liubo02): It's supported by https://github.com/kubernetes-sigs/controller-tools/pull/1179
 //
-// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI;SessionTokenSigning;ClusterSubdomain;TerminableLogTailer;UseTSOReadyAPI;UseSchedulingReadyAPI;UseTiKVReadyAPI
+// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI;SessionTokenSigning;ClusterSubdomain;TerminableLogTailer;UseTSOReadyAPI;UseSchedulingReadyAPI;UseTiKVReadyAPI;UsePDReadyAPIV2
 // +enum
 type Feature string
 
@@ -57,6 +57,7 @@ const (
 	DisablePDDefaultReadinessProbe      Feature      = "DisablePDDefaultReadinessProbe"
 	DisablePDDefaultReadinessProbeStage FeatureStage = FeatureStageAlpha
 
+	// Deprecated: use UsePDReadyAPIV2
 	// UsePDReadyAPI means use PD's /ready API as the readiness probe.
 	// It requires PD v8.5.2 or later.
 	UsePDReadyAPI      Feature      = "UsePDReadyAPI"
@@ -93,4 +94,8 @@ const (
 	// UseTiKVReadyAPI means use TiKV's /ready API as the readiness probe.
 	UseTiKVReadyAPI      Feature      = "UseTiKVReadyAPI"
 	UseTiKVReadyAPIStage FeatureStage = FeatureStageAlpha
+
+	// UsePDReadyAPIV2 means use PD's /readyz API as the readiness probe.
+	UsePDReadyAPIV2      Feature      = "UsePDReadyAPIV2"
+	UsePDReadyAPIV2Stage FeatureStage = FeatureStageAlpha
 )
diff --git a/manifests/crd/core.pingcap.com_clusters.yaml b/manifests/crd/core.pingcap.com_clusters.yaml
@@ -96,6 +96,7 @@ spec:
                       - UseTSOReadyAPI
                       - UseSchedulingReadyAPI
                       - UseTiKVReadyAPI
+                      - UsePDReadyAPIV2
                       type: string
                   required:
                   - name
@@ -303,6 +304,7 @@ spec:
                       - UseTSOReadyAPI
                       - UseSchedulingReadyAPI
                       - UseTiKVReadyAPI
+                      - UsePDReadyAPIV2
                       type: string
                     stage:
                       type: string
diff --git a/manifests/crd/core.pingcap.com_pdgroups.yaml b/manifests/crd/core.pingcap.com_pdgroups.yaml
@@ -114,6 +114,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_pds.yaml b/manifests/crd/core.pingcap.com_pds.yaml
@@ -90,6 +90,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_replicationworkergroups.yaml b/manifests/crd/core.pingcap.com_replicationworkergroups.yaml
@@ -105,6 +105,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_replicationworkers.yaml b/manifests/crd/core.pingcap.com_replicationworkers.yaml
@@ -90,6 +90,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_schedulergroups.yaml b/manifests/crd/core.pingcap.com_schedulergroups.yaml
@@ -109,6 +109,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_schedulers.yaml b/manifests/crd/core.pingcap.com_schedulers.yaml
@@ -95,6 +95,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_schedulinggroups.yaml b/manifests/crd/core.pingcap.com_schedulinggroups.yaml
@@ -104,6 +104,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_schedulings.yaml b/manifests/crd/core.pingcap.com_schedulings.yaml
@@ -90,6 +90,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_ticdcgroups.yaml b/manifests/crd/core.pingcap.com_ticdcgroups.yaml
@@ -100,6 +100,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_ticdcs.yaml b/manifests/crd/core.pingcap.com_ticdcs.yaml
@@ -87,6 +87,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tidbgroups.yaml b/manifests/crd/core.pingcap.com_tidbgroups.yaml
@@ -103,6 +103,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tidbs.yaml b/manifests/crd/core.pingcap.com_tidbs.yaml
@@ -89,6 +89,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tiflashes.yaml b/manifests/crd/core.pingcap.com_tiflashes.yaml
@@ -95,6 +95,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tiflashgroups.yaml b/manifests/crd/core.pingcap.com_tiflashgroups.yaml
@@ -102,6 +102,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tikvgroups.yaml b/manifests/crd/core.pingcap.com_tikvgroups.yaml
@@ -103,6 +103,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tikvs.yaml b/manifests/crd/core.pingcap.com_tikvs.yaml
@@ -95,6 +95,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tiproxies.yaml b/manifests/crd/core.pingcap.com_tiproxies.yaml
@@ -89,6 +89,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tiproxygroups.yaml b/manifests/crd/core.pingcap.com_tiproxygroups.yaml
@@ -103,6 +103,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tsogroups.yaml b/manifests/crd/core.pingcap.com_tsogroups.yaml
@@ -103,6 +103,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tsos.yaml b/manifests/crd/core.pingcap.com_tsos.yaml
@@ -89,6 +89,7 @@ spec:
                   - UseTSOReadyAPI
                   - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
+                  - UsePDReadyAPIV2
                   type: string
                 type: array
               image:
diff --git a/pkg/controllers/pd/tasks/pod.go b/pkg/controllers/pd/tasks/pod.go
@@ -246,10 +246,14 @@ func newPod(cluster *v1alpha1.Cluster, pd *v1alpha1.PD, g features.Gates, cluste
 	if g.Enabled(metav1alpha1.UsePDReadyAPI) {
 		version := semver.MustParse(pd.Spec.Version)
 		if compatibility.Check(version, compatibility.PDReadyAPI) {
-			pod.Spec.Containers[0].ReadinessProbe = buildPDReadinessProbeWithReadyAPI(cluster, coreutil.PDClientPort(pd))
+			pod.Spec.Containers[0].ReadinessProbe = buildPDReadinessProbeWithReadyAPI(cluster, coreutil.PDClientPort(pd), false)
 		}
 	}
 
+	if g.Enabled(metav1alpha1.UsePDReadyAPIV2) {
+		pod.Spec.Containers[0].ReadinessProbe = buildPDReadinessProbeWithReadyAPI(cluster, coreutil.PDClientPort(pd), true)
+	}
+
 	if pd.Spec.Overlay != nil {
 		overlay.OverlayPod(pod, pd.Spec.Overlay.Pod)
 	}
@@ -269,7 +273,7 @@ func buildPDReadinessProbe(port int32) *corev1.Probe {
 	}
 }
 
-func buildPDReadinessProbeWithReadyAPI(cluster *v1alpha1.Cluster, port int32) *corev1.Probe {
+func buildPDReadinessProbeWithReadyAPI(cluster *v1alpha1.Cluster, port int32, v2 bool) *corev1.Probe {
 	tlsClusterEnabled := coreutil.IsTLSClusterEnabled(cluster)
 
 	scheme := "http"
@@ -278,6 +282,9 @@ func buildPDReadinessProbeWithReadyAPI(cluster *v1alpha1.Cluster, port int32) *c
 	}
 
 	readinessURL := fmt.Sprintf("%s://127.0.0.1:%d/pd/api/v2/ready", scheme, port)
+	if v2 {
+		readinessURL = fmt.Sprintf("%s://127.0.0.1:%d/pd/api/v2/readyz", scheme, port)
+	}
 	var command []string
 	command = append(command, "curl", readinessURL,
 		// Fail silently (no output at all) on server errors
diff --git a/pkg/features/reload.go b/pkg/features/reload.go
@@ -65,6 +65,9 @@ var unreloadable = map[meta.Feature][]meta.Component{
 	meta.UseTiKVReadyAPI: {
 		meta.ComponentTiKV,
 	},
+	meta.UsePDReadyAPIV2: {
+		meta.ComponentPD,
+	},
 }
 
 func Reloadable(c meta.Component, update, current []meta.Feature) bool {
diff --git a/tests/e2e/pd/pdms.go b/tests/e2e/pd/pdms.go
@@ -19,6 +19,7 @@ import (
 
 	"github.com/onsi/ginkgo/v2"
 
+	metav1alpha1 "github.com/pingcap/tidb-operator/api/v2/meta/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/runtime"
 	"github.com/pingcap/tidb-operator/tests/e2e/data"
 	"github.com/pingcap/tidb-operator/tests/e2e/framework"
@@ -72,4 +73,40 @@ var _ = ginkgo.Describe("PD", label.PD, label.FeaturePDMS, func() {
 			f.WaitForSchedulingGroupReady(ctx, sg)
 		})
 	})
+
+	ginkgo.Context("NextGen PDMS",
+		label.P0,
+		label.KindNextGen,
+		label.Features(
+			metav1alpha1.UsePDReadyAPIV2,
+			metav1alpha1.UseTSOReadyAPI,
+			metav1alpha1.UseSchedulingReadyAPI,
+		),
+		func() {
+			f.SetupCluster(data.WithFeatureGates(
+				metav1alpha1.UseTSOReadyAPI,
+				metav1alpha1.UsePDReadyAPIV2,
+				metav1alpha1.UseSchedulingReadyAPI,
+			))
+			ginkgo.It("support create 3 PD instances, 2 TSO instances, and 2 scheduling instances", func(ctx context.Context) {
+				f.MustCreateS3(ctx)
+				pdg := f.MustCreatePD(ctx,
+					data.WithPDNextGen(),
+					data.WithMSMode(),
+					data.WithReplicas[*runtime.PDGroup](3),
+				)
+				tg := f.MustCreateTSO(ctx,
+					data.WithTSONextGen(),
+					data.WithReplicas[*runtime.TSOGroup](2),
+				)
+				sg := f.MustCreateScheduling(ctx,
+					data.WithSchedulingNextGen(),
+					data.WithReplicas[*runtime.SchedulingGroup](2),
+				)
+
+				f.WaitForPDGroupReady(ctx, pdg)
+				f.WaitForTSOGroupReady(ctx, tg)
+				f.WaitForSchedulingGroupReady(ctx, sg)
+			})
+		})
 })
diff --git a/tests/e2e/suite/availability/pd.go b/tests/e2e/suite/availability/pd.go
diff --git a/tests/e2e/suite/availability/tikv.go b/tests/e2e/suite/availability/tikv.go
diff --git a/tests/e2e/suite/cluster/tls.go b/tests/e2e/suite/cluster/tls.go
