refactor(kubernetes): optimize pod recycling logic and cleanup tests

- Add support for customizing restart timeout via annotation in docs
- Replace PodsToRecycle slice with RecyclingPods set for efficient lookup
- Refactor allocator to skip pods currently recycling in allocation logic
- Remove deprecated canAllocate function and simplify recycling checks
- Update pool reconciler to collect recycling pods and handle recycling in batch
- Refactor handlePodRecycle to process multiple pods and aggregate errors
- Enhance restart tracker logging with timeout and elapsed time details
- Remove outdated controller unit tests and reduce e2e test scope for simplicity

Author: fengcone

kubernetes/README-ZH.md

@@ -29,7 +29,7 @@ Pool 自定义资源维护一个预热的计算资源池，以实现快速沙箱
 - **池容量限制**：通过池范围的最小和最大限制来控制总体资源消耗。
 - **回收策略 (Recycle Policies)**：支持不同的 Pod 回收策略：
   - **Delete (默认)**：Pod 在返回池时会被删除并根据模板重新创建，确保环境绝对纯净。
-  - **Restart**：通过向所有容器的 PID 1 发送 SIGTERM 信号优雅终止进程，并依赖 Kubernetes 的 `restartPolicy` 触发重启。这种方式比 `Delete` 更快，但要求 `PodTemplateSpec` 中的 `restartPolicy` 设置为 `Always`。
+  - **Restart**：通过向所有容器的 PID 1 发送 SIGTERM 信号优雅终止进程，并依赖 Kubernetes 的 `restartPolicy` 触发重启。这种方式比 `Delete` 更快，但要求 `PodTemplateSpec` 中的 `restartPolicy` 设置为 `Always`。可通过 annotation `pool.opensandbox.io/recycle-timeout-sec` 自定义重启超时时间（默认 90 秒）。
 - **自动扩展**：基于当前需求和缓冲区设置进行动态资源分配和释放。
 - **实时状态监控**：显示总数、已分配、可用以及正在重启中的 Pod 数量。
 

kubernetes/README.md

@@ -29,7 +29,7 @@ The Pool custom resource maintains a pool of pre-warmed compute resources to ena
 - **Pool Capacity Limits**: Overall resource consumption control with pool-wide minimum and maximum limits.
 - **Recycle Policies**: Support for different pod recycling strategies:
   - **Delete (Default)**: Pods are deleted and recreated from the template when returned to the pool, ensuring a completely clean environment.
-  - **Restart**: PID 1 in all containers is gracefully terminated (SIGTERM), and the Kubernetes `restartPolicy` triggers a restart. This is faster than `Delete` but requires the `restartPolicy` in `PodTemplateSpec` to be set to `Always`.
+  - **Restart**: PID 1 in all containers is gracefully terminated (SIGTERM), and the Kubernetes `restartPolicy` triggers a restart. This is faster than `Delete` but requires the `restartPolicy` in `PodTemplateSpec` to be set to `Always`. The restart timeout can be customized per-pool via the annotation `pool.opensandbox.io/recycle-timeout-sec` (default: 90s).
 - **Automatic Scaling**: Dynamic resource allocation and deallocation based on current demand and buffer settings.
 - **Real-time Status**: Monitoring of total, allocated, available, and restarting pods.
 

kubernetes/internal/controller/allocator.go

@@ -23,6 +23,7 @@ import (
 	"strconv"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -177,15 +178,15 @@ type AllocSpec struct {
 	Pool *sandboxv1alpha1.Pool
 	// all pods of pool
 	Pods []*corev1.Pod
+
+	RecyclingPods sets.Set[string]
 }
 
 type AllocStatus struct {
 	// PodAllocation maps pod name to sandbox name for currently allocated pods.
 	PodAllocation map[string]string
 	// PodSupplement is the number of additional pods needed to meet sandbox demands.
 	PodSupplement int32
-	// PoolReconciler handles the actual recycle logic (delete or restart).
-	PodsToRecycle []string
 }
 
 type SandboxSyncInfo struct {
@@ -223,7 +224,11 @@ func (allocator *defaultAllocator) Schedule(ctx context.Context, spec *AllocSpec
 		if _, ok := status.PodAllocation[pod.Name]; ok {
 			continue
 		}
-		if !canAllocate(pod) {
+		if spec.RecyclingPods.Has(pod.Name) {
+			continue
+		}
+		// Exclude pods that are restarting (have recycle-meta annotation)
+		if isRecycling(pod) {
 			continue
 		}
 		if pod.Status.Phase != corev1.PodRunning {
@@ -271,7 +276,6 @@ func (allocator *defaultAllocator) initAllocation(ctx context.Context, spec *All
 	var err error
 	status := &AllocStatus{
 		PodAllocation: make(map[string]string),
-		PodsToRecycle: make([]string, 0),
 	}
 	status.PodAllocation, err = allocator.getPodAllocation(ctx, spec.Pool)
 	if err != nil {
@@ -391,7 +395,6 @@ func (allocator *defaultAllocator) deallocate(ctx context.Context, status *Alloc
 		for _, pod := range pods {
 			delete(status.PodAllocation, pod)
 			poolDeallocate = true
-			status.PodsToRecycle = append(status.PodsToRecycle, pod)
 		}
 		delete(sandboxToPods, name)
 	}
@@ -413,7 +416,6 @@ func (allocator *defaultAllocator) doDeallocate(ctx context.Context, status *All
 		if _, ok := status.PodAllocation[pod]; ok {
 			delete(status.PodAllocation, pod)
 			deallocate = true
-			status.PodsToRecycle = append(status.PodsToRecycle, pod)
 		}
 	}
 	pods := make([]string, 0)

kubernetes/internal/controller/allocator_test.go

@@ -109,7 +109,6 @@ func TestAllocatorSchedule(t *testing.T) {
 					"pod2": "sbx2",
 				},
 				PodSupplement: 0,
-				PodsToRecycle: []string{},
 			},
 		},
 		{
@@ -173,7 +172,6 @@ func TestAllocatorSchedule(t *testing.T) {
 					"pod1": "sbx1",
 				},
 				PodSupplement: 1,
-				PodsToRecycle: []string{},
 			},
 		},
 		{
@@ -241,7 +239,6 @@ func TestAllocatorSchedule(t *testing.T) {
 					"pod2": "sbx1",
 				},
 				PodSupplement: 0,
-				PodsToRecycle: []string{},
 			},
 		},
 		{
@@ -296,7 +293,6 @@ func TestAllocatorSchedule(t *testing.T) {
 			wantStatus: &AllocStatus{
 				PodAllocation: map[string]string{},
 				PodSupplement: 0,
-				PodsToRecycle: []string{"pod1"},
 			},
 		},
 		{
@@ -363,7 +359,6 @@ func TestAllocatorSchedule(t *testing.T) {
 					"pod-normal": "sbx1",
 				},
 				PodSupplement: 1, // sbx2 needs a pod but only normal pod available
-				PodsToRecycle: []string{},
 			},
 		},
 	}

kubernetes/internal/controller/apis.go

@@ -87,38 +87,12 @@ func setPodRecycleMeta(obj metav1.Object, meta *PodRecycleMeta) {
 	obj.GetAnnotations()[AnnoPodRecycleMeta] = utils.DumpJSON(meta)
 }
 
-// canAllocate checks if a pod is eligible for allocation.
-// A pod can be allocated if:
-// 1. No deallocated-from label (normal pod), OR
-// 2. Has recycle-confirmed label AND no recycle-meta annotation (recycling completed)
-func canAllocate(pod *corev1.Pod) bool {
-	deallocatedFrom := pod.Labels[LabelPodDeallocatedFrom]
-	if deallocatedFrom == "" {
-		return true // Normal pod, no deallocation marker
-	}
-
-	// Has deallocated-from, check if recycling is confirmed and completed
-	recycleConfirmed := pod.Labels[LabelPodRecycleConfirmed]
-	meta := pod.Annotations[AnnoPodRecycleMeta]
-
-	// Can allocate only if recycling is confirmed AND not in restarting state
-	return recycleConfirmed != "" && meta == ""
+func isRestarting(pod *corev1.Pod) bool {
+	return pod.Annotations[AnnoPodRecycleMeta] != ""
 }
 
-func isRestarting(pod *corev1.Pod) bool {
-	// - recycle-confirmed is set when restart starts
-	// - recycle-confirmed is KEPT as a receipt after restart completes
-	// - recycle-meta is cleared when restart completes
-	meta := pod.Annotations[AnnoPodRecycleMeta]
-	if meta == "" {
-		return false
-	}
-	// Parse to verify it's in Restarting state (not just stale data)
-	var recycleMeta PodRecycleMeta
-	if err := json.Unmarshal([]byte(meta), &recycleMeta); err != nil {
-		return false
-	}
-	return recycleMeta.State == RecycleStateRestarting
+func isRecycling(pod *corev1.Pod) bool {
+	return pod.Labels[LabelPodDeallocatedFrom] != "" || pod.Annotations[AnnoPodRecycleMeta] != ""
 }
 
 // AnnotationSandboxEndpoints Use the exported constant from pkg/utils