feat(scheduling): support health api (#6516)

liubog2008 · web-flow · commit b38353a2ed7a · 2025-10-23T10:30:10.000Z
diff --git a/api/meta/v1alpha1/feature.go b/api/meta/v1alpha1/feature.go
@@ -18,7 +18,7 @@ package v1alpha1
 // NOTE(liubo02): +enum is not supported now, we have to add all enum into comments
 // NOTE(liubo02): It's supported by https://github.com/kubernetes-sigs/controller-tools/pull/1179
 //
-// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI;SessionTokenSigning;ClusterSubdomain;TerminableLogTailer;UseTSOReadyAPI;UseTiKVReadyAPI
+// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI;SessionTokenSigning;ClusterSubdomain;TerminableLogTailer;UseTSOReadyAPI;UseSchedulingReadyAPI;UseTiKVReadyAPI
 // +enum
 type Feature string
 
@@ -86,6 +86,10 @@ const (
 	UseTSOReadyAPI      Feature      = "UseTSOReadyAPI"
 	UseTSOReadyAPIStage FeatureStage = FeatureStageAlpha
 
+	// UseSchedulingReadyAPI calls /health api to check readiness for scheduling pods.
+	UseSchedulingReadyAPI      Feature      = "UseSchedulingReadyAPI"
+	UseSchedulingReadyAPIStage FeatureStage = FeatureStageAlpha
+
 	// UseTiKVReadyAPI means use TiKV's /ready API as the readiness probe.
 	UseTiKVReadyAPI      Feature      = "UseTiKVReadyAPI"
 	UseTiKVReadyAPIStage FeatureStage = FeatureStageAlpha
diff --git a/manifests/crd/core.pingcap.com_clusters.yaml b/manifests/crd/core.pingcap.com_clusters.yaml
@@ -94,6 +94,7 @@ spec:
                       - ClusterSubdomain
                       - TerminableLogTailer
                       - UseTSOReadyAPI
+                      - UseSchedulingReadyAPI
                       - UseTiKVReadyAPI
                       type: string
                   required:
@@ -300,6 +301,7 @@ spec:
                       - ClusterSubdomain
                       - TerminableLogTailer
                       - UseTSOReadyAPI
+                      - UseSchedulingReadyAPI
                       - UseTiKVReadyAPI
                       type: string
                     stage:
diff --git a/manifests/crd/core.pingcap.com_pdgroups.yaml b/manifests/crd/core.pingcap.com_pdgroups.yaml
@@ -112,6 +112,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_pds.yaml b/manifests/crd/core.pingcap.com_pds.yaml
@@ -88,6 +88,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_replicationworkergroups.yaml b/manifests/crd/core.pingcap.com_replicationworkergroups.yaml
@@ -103,6 +103,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_replicationworkers.yaml b/manifests/crd/core.pingcap.com_replicationworkers.yaml
@@ -88,6 +88,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_schedulergroups.yaml b/manifests/crd/core.pingcap.com_schedulergroups.yaml
@@ -107,6 +107,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_schedulers.yaml b/manifests/crd/core.pingcap.com_schedulers.yaml
@@ -93,6 +93,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_schedulinggroups.yaml b/manifests/crd/core.pingcap.com_schedulinggroups.yaml
@@ -102,6 +102,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_schedulings.yaml b/manifests/crd/core.pingcap.com_schedulings.yaml
@@ -88,6 +88,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_ticdcgroups.yaml b/manifests/crd/core.pingcap.com_ticdcgroups.yaml
@@ -98,6 +98,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_ticdcs.yaml b/manifests/crd/core.pingcap.com_ticdcs.yaml
@@ -85,6 +85,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tidbgroups.yaml b/manifests/crd/core.pingcap.com_tidbgroups.yaml
@@ -101,6 +101,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tidbs.yaml b/manifests/crd/core.pingcap.com_tidbs.yaml
@@ -87,6 +87,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tiflashes.yaml b/manifests/crd/core.pingcap.com_tiflashes.yaml
@@ -93,6 +93,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tiflashgroups.yaml b/manifests/crd/core.pingcap.com_tiflashgroups.yaml
@@ -100,6 +100,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tikvgroups.yaml b/manifests/crd/core.pingcap.com_tikvgroups.yaml
@@ -101,6 +101,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tikvs.yaml b/manifests/crd/core.pingcap.com_tikvs.yaml
@@ -93,6 +93,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tiproxies.yaml b/manifests/crd/core.pingcap.com_tiproxies.yaml
@@ -87,6 +87,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tiproxygroups.yaml b/manifests/crd/core.pingcap.com_tiproxygroups.yaml
@@ -101,6 +101,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tsogroups.yaml b/manifests/crd/core.pingcap.com_tsogroups.yaml
@@ -101,6 +101,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/manifests/crd/core.pingcap.com_tsos.yaml b/manifests/crd/core.pingcap.com_tsos.yaml
@@ -87,6 +87,7 @@ spec:
                   - ClusterSubdomain
                   - TerminableLogTailer
                   - UseTSOReadyAPI
+                  - UseSchedulingReadyAPI
                   - UseTiKVReadyAPI
                   type: string
                 type: array
diff --git a/pkg/controllers/scheduling/tasks/pod.go b/pkg/controllers/scheduling/tasks/pod.go
@@ -16,15 +16,19 @@ package tasks
 
 import (
 	"context"
+	"fmt"
+	"path"
 	"path/filepath"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
+	metav1alpha1 "github.com/pingcap/tidb-operator/api/v2/meta/v1alpha1"
 	coreutil "github.com/pingcap/tidb-operator/pkg/apiutil/core/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/client"
+	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/image"
 	"github.com/pingcap/tidb-operator/pkg/overlay"
 	"github.com/pingcap/tidb-operator/pkg/reloadable"
@@ -43,7 +47,7 @@ func TaskPod(state *ReconcileContext, c client.Client) task.Task {
 		ck := state.Cluster()
 		obj := state.Object()
 		logger := logr.FromContextOrDiscard(ctx)
-		expected := newPod(ck, obj)
+		expected := newPod(ck, obj, state.FeatureGates())
 		pod := state.Pod()
 		if pod == nil {
 			if err := c.Apply(ctx, expected); err != nil {
@@ -76,7 +80,7 @@ func TaskPod(state *ReconcileContext, c client.Client) task.Task {
 	})
 }
 
-func newPod(cluster *v1alpha1.Cluster, scheduling *v1alpha1.Scheduling) *corev1.Pod {
+func newPod(cluster *v1alpha1.Cluster, scheduling *v1alpha1.Scheduling, fg features.Gates) *corev1.Pod {
 	vols := []corev1.Volume{
 		{
 			Name: v1alpha1.VolumeNameConfig,
@@ -163,10 +167,50 @@ func newPod(cluster *v1alpha1.Cluster, scheduling *v1alpha1.Scheduling) *corev1.
 		},
 	}
 
+	if fg.Enabled(metav1alpha1.UseSchedulingReadyAPI) {
+		pod.Spec.Containers[0].ReadinessProbe = buildReadinessProbe(cluster, coreutil.SchedulingClientPort(scheduling))
+	}
+
 	if scheduling.Spec.Overlay != nil {
 		overlay.OverlayPod(pod, scheduling.Spec.Overlay.Pod)
 	}
 
 	reloadable.MustEncodeLastSchedulingTemplate(scheduling, pod)
 	return pod
 }
+
+func buildReadinessProbe(cluster *v1alpha1.Cluster, port int32) *corev1.Probe {
+	tlsClusterEnabled := coreutil.IsTLSClusterEnabled(cluster)
+
+	scheme := "http"
+	if tlsClusterEnabled {
+		scheme = "https"
+	}
+
+	readinessURL := fmt.Sprintf("%s://127.0.0.1:%d/health", scheme, port)
+	var command []string
+	command = append(command, "curl", readinessURL,
+		// Fail silently (no output at all) on server errors
+		// without this if the server return 500, the exist code will be 0
+		// and probe is success.
+		"--fail",
+		// Silent mode, only print error
+		"-sS",
+		// follow 301 or 302 redirect
+		"--location")
+
+	if tlsClusterEnabled {
+		cacert := path.Join(v1alpha1.DirPathClusterTLSScheduling, corev1.ServiceAccountRootCAKey)
+		cert := path.Join(v1alpha1.DirPathClusterTLSScheduling, corev1.TLSCertKey)
+		key := path.Join(v1alpha1.DirPathClusterTLSScheduling, corev1.TLSPrivateKeyKey)
+		command = append(command, "--cacert", cacert, "--cert", cert, "--key", key)
+	}
+
+	return &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			Exec: &corev1.ExecAction{
+				Command: command,
+			},
+		},
+	}
+}
diff --git a/pkg/features/reload.go b/pkg/features/reload.go
@@ -59,6 +59,9 @@ var unreloadable = map[meta.Feature][]meta.Component{
 	meta.UseTSOReadyAPI: {
 		meta.ComponentTSO,
 	},
+	meta.UseSchedulingReadyAPI: {
+		meta.ComponentScheduling,
+	},
 	meta.UseTiKVReadyAPI: {
 		meta.ComponentTiKV,
 	},
diff --git a/tests/e2e/data/tiflash.go b/tests/e2e/data/tiflash.go
@@ -105,6 +105,12 @@ type FlashConfig struct {
 
 func withTiFlashMode(mode string) GroupPatch[*runtime.TiFlashGroup] {
 	return func(obj *runtime.TiFlashGroup) {
+		if mode == "tiflash_write" {
+			obj.Name = obj.Name + "-wn"
+		}
+		if mode == "tiflash_compute" {
+			obj.Name = obj.Name + "-cn"
+		}
 		c := TiFlashConfig{}
 		d, e := toml.Codec[TiFlashConfig]()
 		if err := d.Decode([]byte(obj.Spec.Template.Spec.Config), &c); err != nil {
diff --git a/tests/e2e/suite/availability/tiflash.go b/tests/e2e/suite/availability/tiflash.go
@@ -87,13 +87,11 @@ var _ = ginkgo.Describe("TiFlash Availability Test", label.TiFlash, label.Update
 			pdg := f.MustCreatePD(ctx, data.WithPDNextGen())
 			kvg := f.MustCreateTiKV(ctx, data.WithTiKVNextGen())
 			fgc := f.MustCreateTiFlash(ctx,
-				data.WithGroupName[*runtime.TiFlashGroup]("fg-compute"),
 				data.WithReplicas[*runtime.TiFlashGroup](2),
 				data.WithTiFlashNextGen(),
 				data.WithTiFlashComputeMode(),
 			)
 			fgw := f.MustCreateTiFlash(ctx,
-				data.WithGroupName[*runtime.TiFlashGroup]("fg-write"),
 				data.WithReplicas[*runtime.TiFlashGroup](2),
 				data.WithTiFlashNextGen(),
 				data.WithTiFlashWriteMode(),
diff --git a/tests/e2e/suite/cluster/tls.go b/tests/e2e/suite/cluster/tls.go
