Skip to content

Commit 57b42a0

Browse files
authored
Merge pull request #744 from Generalwin/k8s-pool-restart
feat(k8s): Add k8s pool restart , close #452
2 parents c51baf9 + 55f9492 commit 57b42a0

39 files changed

Lines changed: 6151 additions & 726 deletions

kubernetes/apis/sandbox/v1alpha1/pool_types.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,28 @@ import (
2323
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
2424
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
2525

26+
// RecycleType defines the type of recycle policy.
27+
type RecycleType string
28+
29+
const (
30+
// RecycleTypeNoop does nothing when a pod is returned to the pool.
31+
RecycleTypeNoop RecycleType = "Noop"
32+
// RecycleTypeDelete deletes the pod when it is returned to the pool.
33+
RecycleTypeDelete RecycleType = "Delete"
34+
// RecycleTypeRestart restarts the pod containers when it is returned to the pool.
35+
RecycleTypeRestart RecycleType = "Restart"
36+
)
37+
38+
// RecycleStrategy controls how pods are handled when returned to the pool.
39+
type RecycleStrategy struct {
40+
// Type specifies the recycle policy type.
41+
// Default is Delete.
42+
// +kubebuilder:validation:Enum=Delete;Restart;Noop
43+
// +kubebuilder:default=Delete
44+
// +optional
45+
Type RecycleType `json:"type,omitempty"`
46+
}
47+
2648
// PoolSpec defines the desired state of Pool.
2749
type PoolSpec struct {
2850
// Pod Template used to create pre-warmed nodes in the pool.
@@ -39,6 +61,11 @@ type PoolSpec struct {
3961
// UpdateStrategy controls how pool pods are updated when the template changes.
4062
// +optional
4163
UpdateStrategy *UpdateStrategy `json:"updateStrategy,omitempty"`
64+
// RecycleStrategy controls how pods are handled when returned to the pool.
65+
// Default is Delete, which deletes the pod.
66+
// Restart strategy restarts the pod containers instead of deleting.
67+
// +optional
68+
RecycleStrategy *RecycleStrategy `json:"recycleStrategy,omitempty"`
4269
}
4370

4471
type CapacitySpec struct {

kubernetes/apis/sandbox/v1alpha1/zz_generated.deepcopy.go

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

kubernetes/cmd/controller/main.go

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -365,17 +365,11 @@ func main() {
365365
HealthProbeBindAddress: probeAddr,
366366
LeaderElection: enableLeaderElection,
367367
LeaderElectionID: "2fa1c467.opensandbox.io",
368-
// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
369-
// when the Manager ends. This requires the binary to immediately end when the
370-
// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
371-
// speeds up voluntary leader transitions as the new leader don't have to wait
372-
// LeaseDuration time first.
373-
//
374-
// In the default scaffold provided, the program ends immediately after
375-
// the manager stops, so would be fine to enable this option. However,
376-
// if you are doing or is intended to do any operation such as perform cleanups
377-
// after the manager stops then its usage might be unsafe.
378-
// LeaderElectionReleaseOnCancel: true,
368+
// LeaderElectionReleaseOnCancel causes the leader to voluntarily release the lease
369+
// when the Manager is stopped, allowing a new leader to acquire it without waiting
370+
// for the full LeaseDuration. This is safe because main() exits immediately after
371+
// mgr.Start() returns and performs no post-stop cleanup.
372+
LeaderElectionReleaseOnCancel: true,
379373
})
380374
if err != nil {
381375
setupLog.Error(err, "unable to start manager")
@@ -404,10 +398,11 @@ func main() {
404398
os.Exit(1)
405399
}
406400
if err := (&controller.PoolReconciler{
407-
Client: mgr.GetClient(),
408-
Scheme: mgr.GetScheme(),
409-
Recorder: mgr.GetEventRecorderFor("pool-controller"),
410-
Allocator: controller.NewDefaultAllocator(mgr.GetClient()),
401+
Client: mgr.GetClient(),
402+
Scheme: mgr.GetScheme(),
403+
Recorder: mgr.GetEventRecorderFor("pool-controller"),
404+
Allocator: controller.NewDefaultAllocator(mgr.GetClient()),
405+
RestConfig: mgr.GetConfig(),
411406
}).SetupWithManager(mgr, poolConcurrency); err != nil {
412407
setupLog.Error(err, "unable to create controller", "controller", "Pool")
413408
os.Exit(1)

kubernetes/config/crd/bases/sandbox.opensandbox.io_pools.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,23 @@ spec:
9191
- poolMax
9292
- poolMin
9393
type: object
94+
recycleStrategy:
95+
description: |-
96+
RecycleStrategy controls how pods are handled when returned to the pool.
97+
Default is Delete, which deletes the pod.
98+
Restart strategy restarts the pod containers instead of deleting.
99+
properties:
100+
type:
101+
default: Delete
102+
description: |-
103+
Type specifies the recycle policy type.
104+
Default is Delete.
105+
enum:
106+
- Delete
107+
- Restart
108+
- Noop
109+
type: string
110+
type: object
94111
scaleStrategy:
95112
description: ScaleStrategy controls the scaling behavior.
96113
properties:

kubernetes/config/rbac/role.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ rules:
1717
- patch
1818
- update
1919
- watch
20+
- apiGroups:
21+
- ""
22+
resources:
23+
- pods/exec
24+
verbs:
25+
- create
2026
- apiGroups:
2127
- ""
2228
resources:
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
apiVersion: sandbox.opensandbox.io/v1alpha1
2+
kind: Pool
3+
metadata:
4+
labels:
5+
app.kubernetes.io/managed-by: kustomize
6+
app.kubernetes.io/name: sandbox-k8s
7+
name: pool-sample
8+
namespace: opensandbox
9+
spec:
10+
capacitySpec:
11+
bufferMax: 3
12+
bufferMin: 1
13+
poolMax: 5
14+
poolMin: 1
15+
recycleStrategy:
16+
type: Restart
17+
template:
18+
metadata:
19+
labels:
20+
app: example
21+
spec:
22+
containers:
23+
- command:
24+
- /bin/sh
25+
- -c
26+
- |
27+
exec /opt/opensandbox/bin/task-executor -listen-addr=0.0.0.0:5758 >/tmp/task-executor.log 2>&1
28+
env:
29+
- name: SANDBOX_MAIN_CONTAINER
30+
value: main
31+
- name: EXECD_ENVS
32+
value: /opt/opensandbox/.env
33+
- name: EXECD
34+
value: /opt/opensandbox/bin/execd
35+
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/code-interpreter:v1.0.2
36+
name: sandbox
37+
volumeMounts:
38+
- mountPath: /var/lib/sandbox
39+
name: sandbox-storage
40+
- mountPath: /opt/opensandbox/bin
41+
name: opensandbox-bin
42+
initContainers:
43+
- args:
44+
- "cp /workspace/server /opt/opensandbox/bin/task-executor && \nchmod +x /opt/opensandbox/bin/task-executor\n"
45+
command:
46+
- /bin/sh
47+
- -c
48+
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/task-executor:v0.1.0
49+
name: task-executor-installer
50+
volumeMounts:
51+
- mountPath: /opt/opensandbox/bin
52+
name: opensandbox-bin
53+
- args:
54+
- "cp ./execd /opt/opensandbox/bin/execd && \ncp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh
55+
&&\nchmod +x /opt/opensandbox/bin/execd &&\nchmod +x /opt/opensandbox/bin/bootstrap.sh\n"
56+
command:
57+
- /bin/sh
58+
- -c
59+
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.11
60+
name: execd-installer
61+
volumeMounts:
62+
- mountPath: /opt/opensandbox/bin
63+
name: opensandbox-bin
64+
tolerations:
65+
- operator: Exists
66+
volumes:
67+
- emptyDir: {}
68+
name: sandbox-storage
69+
- emptyDir: {}
70+
name: opensandbox-bin

kubernetes/go.mod

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ require (
1717

1818
require github.com/cenkalti/backoff/v5 v5.0.3 // indirect
1919

20+
require (
21+
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
22+
github.com/moby/spdystream v0.5.0 // indirect
23+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
24+
)
25+
2026
require (
2127
cel.dev/expr v0.25.1 // indirect
2228
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect

kubernetes/go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
22
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
33
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
44
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
5+
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
6+
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
57
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
68
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
79
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
@@ -66,6 +68,8 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY
6668
github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
6769
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
6870
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
71+
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
72+
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
6973
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
7074
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
7175
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -89,13 +93,17 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
8993
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
9094
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
9195
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
96+
github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU=
97+
github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
9298
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
9399
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
94100
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
95101
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
96102
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
97103
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
98104
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
105+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
106+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
99107
github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
100108
github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
101109
github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw=
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright 2025 Alibaba Group Holding Ltd.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package algorithm
16+
17+
// Algorithm determines how available pods are distributed among sandbox requests.
18+
type Algorithm interface {
19+
// Schedule distributes available pods among sandbox requests and returns the allocation action.
20+
Schedule(availablePods []string, allRequest []*SandboxRequest) *AllocAction
21+
}
22+
23+
// SandboxRequest describes a sandbox's allocation need.
24+
type SandboxRequest struct {
25+
SandboxName string
26+
CurAllocation []string
27+
CurReleased []string
28+
PodSupplement int32
29+
ToRelease []string
30+
}
31+
32+
// AllocAction represents the result of a scheduling decision.
33+
type AllocAction struct {
34+
// allocate pods to sandbox (sandbox -> pods)
35+
ToAllocate map[string][]string
36+
// release pods from sandbox (sandbox -> pods)
37+
ToRelease map[string][]string
38+
// pod request count
39+
PodSupplement int32
40+
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright 2025 Alibaba Group Holding Ltd.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package algorithm
16+
17+
// PackedSchedule allocates pods to each sandbox in order, fully satisfying one before moving to the next.
18+
// This is the default algorithm and provides the simplest packing strategy.
19+
type PackedSchedule struct{}
20+
21+
func (p *PackedSchedule) Schedule(availablePods []string, allRequest []*SandboxRequest) *AllocAction {
22+
action := &AllocAction{
23+
ToAllocate: make(map[string][]string),
24+
ToRelease: make(map[string][]string),
25+
PodSupplement: int32(0),
26+
}
27+
28+
for _, req := range allRequest {
29+
if len(req.ToRelease) > 0 {
30+
action.ToRelease[req.SandboxName] = req.ToRelease
31+
}
32+
33+
need := req.PodSupplement
34+
if need <= 0 {
35+
continue
36+
}
37+
if int32(len(availablePods)) >= need {
38+
action.ToAllocate[req.SandboxName] = availablePods[:need]
39+
availablePods = availablePods[need:]
40+
} else if len(availablePods) > 0 {
41+
action.ToAllocate[req.SandboxName] = availablePods
42+
action.PodSupplement += need - int32(len(availablePods))
43+
availablePods = nil
44+
} else {
45+
action.PodSupplement += need
46+
}
47+
}
48+
49+
return action
50+
}

0 commit comments

Comments
 (0)