refactor(pool): improve pod recycle and allocation logic with finalizers

- Add FinalizerPoolRecycle for pool mode BatchSandbox with restart policy
- Implement ensureFinalizer helper to manage finalizers robustly
- Handle pool recycle process before task cleanup on BatchSandbox deletion
- Enhance canAllocate to exclude pods not ready after recycle confirmation
- Modify handlePodRecycle to support restart timeout from pool annotations
- Adjust PoolReconciler to process pod recycle before scheduling and allocation
- Introduce needsRecycleConfirmation to detect pods needing recycle handling
- Count recycling pods in pool scaling decisions instead of restarting pods
- Update allocator to skip pods that cannot allocate (e.g., still recycling)
- Add unit tests for canAllocate logic on pod labels and annotations
- Update e2e test to verify Delete policy deletes pods and pool replenishment
- Remove deprecated InitialRestartCounts from PodRecycleMeta for clarity
- Refactor restartTracker to remove embedded restartTimeout field
- Update restartTracker HandleRestart call to accept timeout parameter
- Clean up logging and error handling for finalizer and pod recycle operations

Author: fengcone

kubernetes/cmd/controller/main.go

@@ -266,13 +266,14 @@ func main() {
 		os.Exit(1)
 	}
 	kubeClient := kubernetes.NewForConfigOrDie(mgr.GetConfig())
-	restartTracker := controller.NewRestartTracker(mgr.GetClient(), kubeClient, mgr.GetConfig(), restartTimeout)
+	restartTracker := controller.NewRestartTracker(mgr.GetClient(), kubeClient, mgr.GetConfig())
 	if err := (&controller.PoolReconciler{
 		Client:         mgr.GetClient(),
 		Scheme:         mgr.GetScheme(),
 		Recorder:       mgr.GetEventRecorderFor("pool-controller"),
 		Allocator:      controller.NewDefaultAllocator(mgr.GetClient()),
 		RestartTracker: restartTracker,
+		RestartTimeout: restartTimeout,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Pool")
 		os.Exit(1)

kubernetes/internal/controller/allocator.go

@@ -214,7 +214,6 @@ func NewDefaultAllocator(client client.Client) Allocator {
 
 func (allocator *defaultAllocator) Schedule(ctx context.Context, spec *AllocSpec) (*AllocStatus, []SandboxSyncInfo, bool, error) {
 	log := logf.FromContext(ctx)
-	log.Info("Schedule started", "pool", spec.Pool.Name, "totalPods", len(spec.Pods), "sandboxes", len(spec.Sandboxes))
 	status, err := allocator.initAllocation(ctx, spec)
 	if err != nil {
 		return nil, nil, false, err
@@ -224,15 +223,14 @@ func (allocator *defaultAllocator) Schedule(ctx context.Context, spec *AllocSpec
 		if _, ok := status.PodAllocation[pod.Name]; ok {
 			continue
 		}
-		if pod.Status.Phase != corev1.PodRunning {
+		if !canAllocate(pod) {
 			continue
 		}
-		if isRestarting(pod) {
+		if pod.Status.Phase != corev1.PodRunning {
 			continue
 		}
 		availablePods = append(availablePods, pod.Name)
 	}
-	log.V(1).Info("Schedule init", "existingAllocations", len(status.PodAllocation), "availablePods", len(availablePods))
 	sandboxToPods := make(map[string][]string)
 	for podName, sandboxName := range status.PodAllocation {
 		sandboxToPods[sandboxName] = append(sandboxToPods[sandboxName], podName)

kubernetes/internal/controller/allocator_test.go

@@ -299,6 +299,73 @@ func TestAllocatorSchedule(t *testing.T) {
 				PodsToRecycle: []string{"pod1"},
 			},
 		},
+		{
+			name: "pod with deallocated-from label is excluded",
+			spec: &AllocSpec{
+				Pods: []*corev1.Pod{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "pod-normal",
+						},
+						Status: corev1.PodStatus{
+							Phase: corev1.PodRunning,
+						},
+					},
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "pod-deallocated",
+							Labels: map[string]string{
+								"pool.opensandbox.io/deallocated-from": "bsx-uid-123",
+							},
+						},
+						Status: corev1.PodStatus{
+							Phase: corev1.PodRunning,
+						},
+					},
+				},
+				Pool: &sandboxv1alpha1.Pool{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "pool1",
+					},
+				},
+				Sandboxes: []*sandboxv1alpha1.BatchSandbox{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "sbx1",
+						},
+						Spec: sandboxv1alpha1.BatchSandboxSpec{
+							PoolRef:  "pool1",
+							Replicas: &replica1,
+						},
+					},
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "sbx2",
+						},
+						Spec: sandboxv1alpha1.BatchSandboxSpec{
+							PoolRef:  "pool1",
+							Replicas: &replica1,
+						},
+					},
+				},
+			},
+			poolAlloc: &PoolAllocation{
+				PodAllocation: map[string]string{},
+			},
+			sandboxAlloc: &SandboxAllocation{
+				Pods: []string{},
+			},
+			release: &AllocationRelease{
+				Pods: []string{},
+			},
+			wantStatus: &AllocStatus{
+				PodAllocation: map[string]string{
+					"pod-normal": "sbx1",
+				},
+				PodSupplement: 1, // sbx2 needs a pod but only normal pod available
+				PodsToRecycle: []string{},
+			},
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {

kubernetes/internal/controller/apis.go

@@ -36,6 +36,15 @@ const (
 	AnnoPodRecycleMeta = "pool.opensandbox.io/recycle-meta"
 
 	FinalizerTaskCleanup = "batch-sandbox.sandbox.opensandbox.io/task-cleanup"
+	FinalizerPoolRecycle = "batch-sandbox.sandbox.opensandbox.io/pool-recycle"
+
+	// Value is the BatchSandbox UID.
+	LabelPodDeallocatedFrom = "pool.opensandbox.io/deallocated-from"
+	// LabelPodRecycleConfirmed marks that Pool has confirmed recycling.
+	// Value is the BatchSandbox UID from deallocated-from label.
+	LabelPodRecycleConfirmed = "pool.opensandbox.io/recycle-confirmed"
+
+	LabelPodRecycleTimeoutSec = "pool.opensandbox.io/recycle-timeout-sec"
 )
 
 // PodRecycleState defines the state of Pod recycle.
@@ -57,10 +66,6 @@ type PodRecycleMeta struct {
 
 	// TriggeredAt: Restart trigger timestamp (milliseconds)
 	TriggeredAt int64 `json:"triggeredAt"`
-
-	// InitialRestartCounts: Restart counts of containers when restart was triggered.
-	// Used to verify that containers have actually restarted in this cycle.
-	InitialRestartCounts map[string]int32 `json:"initialRestartCounts,omitempty"`
 }
 
 // parsePodRecycleMeta parses the recycle metadata from Pod annotations.
@@ -82,12 +87,38 @@ func setPodRecycleMeta(obj metav1.Object, meta *PodRecycleMeta) {
 	obj.GetAnnotations()[AnnoPodRecycleMeta] = utils.DumpJSON(meta)
 }
 
+// canAllocate checks if a pod is eligible for allocation.
+// A pod can be allocated if:
+// 1. No deallocated-from label (normal pod), OR
+// 2. Has recycle-confirmed label AND no recycle-meta annotation (recycling completed)
+func canAllocate(pod *corev1.Pod) bool {
+	deallocatedFrom := pod.Labels[LabelPodDeallocatedFrom]
+	if deallocatedFrom == "" {
+		return true // Normal pod, no deallocation marker
+	}
+
+	// Has deallocated-from, check if recycling is confirmed and completed
+	recycleConfirmed := pod.Labels[LabelPodRecycleConfirmed]
+	meta := pod.Annotations[AnnoPodRecycleMeta]
+
+	// Can allocate only if recycling is confirmed AND not in restarting state
+	return recycleConfirmed != "" && meta == ""
+}
+
 func isRestarting(pod *corev1.Pod) bool {
-	meta, err := parsePodRecycleMeta(pod)
-	if err != nil {
+	// - recycle-confirmed is set when restart starts
+	// - recycle-confirmed is KEPT as a receipt after restart completes
+	// - recycle-meta is cleared when restart completes
+	meta := pod.Annotations[AnnoPodRecycleMeta]
+	if meta == "" {
+		return false
+	}
+	// Parse to verify it's in Restarting state (not just stale data)
+	var recycleMeta PodRecycleMeta
+	if err := json.Unmarshal([]byte(meta), &recycleMeta); err != nil {
 		return false
 	}
-	return meta.State == RecycleStateRestarting
+	return recycleMeta.State == RecycleStateRestarting
 }
 
 // AnnotationSandboxEndpoints Use the exported constant from pkg/utils

kubernetes/internal/controller/apis_test.go

@@ -0,0 +1,101 @@
+// Copyright 2025 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controller
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestCanAllocate(t *testing.T) {
+	tests := []struct {
+		name     string
+		pod      *corev1.Pod
+		expected bool
+	}{
+		{
+			name: "normal pod without labels",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pod-normal",
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "pod with deallocated-from but no recycle-confirmed",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pod-deallocated",
+					Labels: map[string]string{
+						LabelPodDeallocatedFrom: "bsx-uid-123",
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "pod with deallocated-from and recycle-confirmed, no recycle-meta",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pod-confirmed",
+					Labels: map[string]string{
+						LabelPodDeallocatedFrom:  "bsx-uid-123",
+						LabelPodRecycleConfirmed: "bsx-uid-123",
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "pod with deallocated-from and recycle-confirmed and recycle-meta (still restarting)",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pod-restarting",
+					Labels: map[string]string{
+						LabelPodDeallocatedFrom:  "bsx-uid-123",
+						LabelPodRecycleConfirmed: "bsx-uid-123",
+					},
+					Annotations: map[string]string{
+						AnnoPodRecycleMeta: `{"state":"Restarting","triggeredAt":1234567890}`,
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "pod with only recycle-confirmed (edge case)",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pod-only-confirmed",
+					Labels: map[string]string{
+						LabelPodRecycleConfirmed: "bsx-uid-123",
+					},
+				},
+			},
+			expected: true, // No deallocated-from means normal pod
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := canAllocate(tt.pod)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}