Skip to content

Commit 0e76c87

Browse files
authored
feat: add automatic defragmentation for etcd (#127)
1 parent 57ceb8b commit 0e76c87

8 files changed

Lines changed: 336 additions & 62 deletions

File tree

api/v1alpha1/hostedcontrolplane_types.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,10 @@ type HostedControlPlaneStatus struct {
279279
ETCDLastBackupTime metav1.Time `json:"etcdLastBackupTime,omitempty"`
280280
//+kubebuilder:validation:Optional
281281
ETCDNextBackupTime metav1.Time `json:"etcdNextBackupTime,omitempty"`
282+
//+kubebuilder:validation:Optional
283+
ETCDLastDefragTime metav1.Time `json:"etcdLastDefragTime,omitempty"`
284+
//+kubebuilder:validation:Optional
285+
ETCDLastAttemptedDefragTime metav1.Time `json:"etcdLastAttemptedDefragTime,omitempty"`
282286

283287
// Required fields by CAPI
284288
// https://cluster-api.sigs.k8s.io/developer/providers/contracts/control-plane#controlplane-replicas

api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

go.mod

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ require (
193193
k8s.io/cri-api v0.35.0 // indirect
194194
k8s.io/cri-client v0.35.0 // indirect
195195
k8s.io/klog/v2 v2.130.1 // indirect
196-
k8s.io/kms v0.35.0 // indirect
196+
k8s.io/kms v0.35.3 // indirect
197197
k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 // indirect
198198
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect
199199
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
@@ -205,15 +205,15 @@ require (
205205
)
206206

207207
replace (
208-
k8s.io/api => k8s.io/api v0.35.0
209-
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.35.0
210-
k8s.io/apimachinery => k8s.io/apimachinery v0.35.0
211-
k8s.io/apiserver => k8s.io/apiserver v0.35.0
212-
k8s.io/cli-runtime => k8s.io/cli-runtime v0.35.0
213-
k8s.io/client-go => k8s.io/client-go v0.35.0
208+
k8s.io/api => k8s.io/api v0.35.3
209+
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.35.3
210+
k8s.io/apimachinery => k8s.io/apimachinery v0.35.3
211+
k8s.io/apiserver => k8s.io/apiserver v0.35.3
212+
k8s.io/cli-runtime => k8s.io/cli-runtime v0.35.3
213+
k8s.io/client-go => k8s.io/client-go v0.35.3
214214
k8s.io/cloud-provider => k8s.io/cloud-provider v0.35.0
215215
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.35.0
216-
k8s.io/component-base => k8s.io/component-base v0.35.0
216+
k8s.io/component-base => k8s.io/component-base v0.35.3
217217
k8s.io/component-helpers => k8s.io/component-helpers v0.35.0
218218
k8s.io/controller-manager => k8s.io/controller-manager v0.35.0
219219
k8s.io/cri-api => k8s.io/cri-api v0.35.0

go.sum

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -494,24 +494,24 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs
494494
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
495495
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
496496
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
497-
k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY=
498-
k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA=
499-
k8s.io/apiextensions-apiserver v0.35.0 h1:3xHk2rTOdWXXJM+RDQZJvdx0yEOgC0FgQ1PlJatA5T4=
500-
k8s.io/apiextensions-apiserver v0.35.0/go.mod h1:E1Ahk9SADaLQ4qtzYFkwUqusXTcaV2uw3l14aqpL2LU=
501-
k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8=
502-
k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=
503-
k8s.io/apiserver v0.35.0 h1:CUGo5o+7hW9GcAEF3x3usT3fX4f9r8xmgQeCBDaOgX4=
504-
k8s.io/apiserver v0.35.0/go.mod h1:QUy1U4+PrzbJaM3XGu2tQ7U9A4udRRo5cyxkFX0GEds=
505-
k8s.io/cli-runtime v0.35.0 h1:PEJtYS/Zr4p20PfZSLCbY6YvaoLrfByd6THQzPworUE=
506-
k8s.io/cli-runtime v0.35.0/go.mod h1:VBRvHzosVAoVdP3XwUQn1Oqkvaa8facnokNkD7jOTMY=
507-
k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE=
508-
k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o=
497+
k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ=
498+
k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4=
499+
k8s.io/apiextensions-apiserver v0.35.3 h1:2fQUhEO7P17sijylbdwt0nBdXP0TvHrHj0KeqHD8FiU=
500+
k8s.io/apiextensions-apiserver v0.35.3/go.mod h1:tK4Kz58ykRpwAEkXUb634HD1ZAegEElktz/B3jgETd8=
501+
k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8=
502+
k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=
503+
k8s.io/apiserver v0.35.3 h1:D2eIcfJ05hEAEewoSDg+05e0aSRwx8Y4Agvd/wiomUI=
504+
k8s.io/apiserver v0.35.3/go.mod h1:JI0n9bHYzSgIxgIrfe21dbduJ9NHzKJ6RchcsmIKWKY=
505+
k8s.io/cli-runtime v0.35.3 h1:UZq4ipNimtzBmhN7PPKbfAdqo8quK0H0UdGl6qAQnqI=
506+
k8s.io/cli-runtime v0.35.3/go.mod h1:O7MUmCqcKSd5xI+O5X7/pRkB5l0O2NIhOdUVwbHLXu4=
507+
k8s.io/client-go v0.35.3 h1:s1lZbpN4uI6IxeTM2cpdtrwHcSOBML1ODNTCCfsP1pg=
508+
k8s.io/client-go v0.35.3/go.mod h1:RzoXkc0mzpWIDvBrRnD+VlfXP+lRzqQjCmKtiwZ8Q9c=
509509
k8s.io/cloud-provider v0.35.0 h1:syiBCQbKh2gho/S1BkIl006Dc44pV8eAtGZmv5NMe7M=
510510
k8s.io/cloud-provider v0.35.0/go.mod h1:7grN+/Nt5Hf7tnSGPT3aErt4K7aQpygyCrGpbrQbzNc=
511511
k8s.io/cluster-bootstrap v0.35.0 h1:VXnil8zw+FikqvytJYLB8wcvjxbUCyqMkiC//k426Y0=
512512
k8s.io/cluster-bootstrap v0.35.0/go.mod h1:X6sjEjVUFSfFNIzJ6VAIuwwh2QiDtsVX1xZgcGX4gD8=
513-
k8s.io/component-base v0.35.0 h1:+yBrOhzri2S1BVqyVSvcM3PtPyx5GUxCK2tinZz1G94=
514-
k8s.io/component-base v0.35.0/go.mod h1:85SCX4UCa6SCFt6p3IKAPej7jSnF3L8EbfSyMZayJR0=
513+
k8s.io/component-base v0.35.3 h1:mbKbzoIMy7JDWS/wqZobYW1JDVRn/RKRaoMQHP9c4P0=
514+
k8s.io/component-base v0.35.3/go.mod h1:IZ8LEG30kPN4Et5NeC7vjNv5aU73ku5MS15iZyvyMYk=
515515
k8s.io/component-helpers v0.35.0 h1:wcXv7HJRksgVjM4VlXJ1CNFBpyDHruRI99RrBtrJceA=
516516
k8s.io/component-helpers v0.35.0/go.mod h1:ahX0m/LTYmu7fL3W8zYiIwnQ/5gT28Ex4o2pymF63Co=
517517
k8s.io/controller-manager v0.35.0 h1:KteodmfVIRzfZ3RDaxhnHb72rswBxEngvdL9vuZOA9A=
@@ -522,8 +522,8 @@ k8s.io/cri-client v0.35.0 h1:U1K4bteO93yioUS38804ybN+kWaon9zrzVtB37I3fCs=
522522
k8s.io/cri-client v0.35.0/go.mod h1:XG5GkuuSpxvungsJVzW58NyWBoGSQhMMJmE5c66m9N8=
523523
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
524524
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
525-
k8s.io/kms v0.35.0 h1:/x87FED2kDSo66csKtcYCEHsxF/DBlNl7LfJ1fVQs1o=
526-
k8s.io/kms v0.35.0/go.mod h1:VT+4ekZAdrZDMgShK37vvlyHUVhwI9t/9tvh0AyCWmQ=
525+
k8s.io/kms v0.35.3 h1:jaxr/7dNqcztGldnfCEZg8DegEOnHV6cfoBC2ACMWEg=
526+
k8s.io/kms v0.35.3/go.mod h1:VT+4ekZAdrZDMgShK37vvlyHUVhwI9t/9tvh0AyCWmQ=
527527
k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 h1:HhDfevmPS+OalTjQRKbTHppRIz01AWi8s45TMXStgYY=
528528
k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ=
529529
k8s.io/kube-proxy v0.35.0 h1:erv2wYmGZ6nyu/FtmaIb+ORD3q2rfZ4Fhn7VXs/8cPQ=

pkg/reconcilers/etcd_cluster/etcd_client/etcd_client.go

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,17 @@ var (
3232
errETCDCAFailedToAppend = errors.New("failed to append etcd CA certificate")
3333
)
3434

35+
const (
36+
etcdDefaultCallTimeout = 10 * time.Second
37+
etcdDefragCallTimeout = 5 * time.Minute
38+
)
39+
3540
type EtcdClient interface {
3641
GetStatuses(ctx context.Context) (map[string]*clientv3.StatusResponse, error)
3742
CreateSnapshot(ctx context.Context) (*clientv3.SnapshotResponse, error)
3843
ListAlarms(ctx context.Context) (*clientv3.AlarmResponse, error)
3944
DisarmAlarm(ctx context.Context, alarm *clientv3.AlarmMember) error
45+
Defragment(ctx context.Context) error
4046
}
4147

4248
type etcdClient struct {
@@ -104,6 +110,7 @@ func (e *etcdClient) GetStatuses(ctx context.Context) (map[string]*clientv3.Stat
104110
e.clientCertificate,
105111
e.endpoints,
106112
e.serverPort,
113+
etcdDefaultCallTimeout,
107114
clientv3.Client.Status,
108115
)
109116
},
@@ -140,6 +147,24 @@ func (e *etcdClient) ListAlarms(ctx context.Context) (*clientv3.AlarmResponse, e
140147
)
141148
}
142149

150+
func (e *etcdClient) Defragment(ctx context.Context) error {
151+
return tracing.WithSpan1(ctx, tracer, "EtcdClient.Defragment",
152+
func(ctx context.Context, span trace.Span) error {
153+
_, err := callETCDFuncOnAllMembers(
154+
ctx,
155+
e.tracerProvider,
156+
e.caPool,
157+
e.clientCertificate,
158+
e.endpoints,
159+
e.serverPort,
160+
etcdDefragCallTimeout,
161+
clientv3.Client.Defragment,
162+
)
163+
return err
164+
},
165+
)
166+
}
167+
143168
func (e *etcdClient) DisarmAlarm(ctx context.Context, alarm *clientv3.AlarmMember) error {
144169
return tracing.WithSpan1(ctx, tracer, "EtcdClient.DisarmAlarm",
145170
func(ctx context.Context, span trace.Span) error {
@@ -172,6 +197,7 @@ func callETCDFuncOnAllMembers[R any](
172197
clientCertificate tls.Certificate,
173198
endpoints map[string]string,
174199
etcdServerPort int32,
200+
callTimeout time.Duration,
175201
etcdFunc func(client clientv3.Client, ctx context.Context, endpoint string) (*R, error),
176202
) (map[string]*R, error) {
177203
return tracing.WithSpan(ctx, tracer, "CallETCDFuncOnAllMembers",
@@ -182,7 +208,15 @@ func callETCDFuncOnAllMembers[R any](
182208
results := make(map[string]*R, len(endpointMap))
183209
var errs error
184210
for _, endpoint := range endpointMap {
185-
result, err := callETCDFuncOnMember(ctx, tracerProvider, endpoint, caPool, clientCertificate, etcdFunc)
211+
result, err := callETCDFuncOnMember(
212+
ctx,
213+
tracerProvider,
214+
endpoint,
215+
caPool,
216+
clientCertificate,
217+
callTimeout,
218+
etcdFunc,
219+
)
186220
errs = errors.Join(errs, err)
187221
results[endpoint] = result
188222
}
@@ -238,6 +272,7 @@ func callETCDFuncOnMember[R any](
238272
endpoint string,
239273
caPool *x509.CertPool,
240274
clientCertificate tls.Certificate,
275+
callTimeout time.Duration,
241276
etcdFunc func(client clientv3.Client, ctx context.Context, endpoint string) (R, error),
242277
) (R, error) {
243278
return tracing.WithSpan(ctx, tracer, "CallETCDFuncOnMember",
@@ -279,7 +314,7 @@ func callETCDFuncOnMember[R any](
279314

280315
var result R
281316
_, _, err = slices.AttemptWithDelay(4, 5*time.Second, func(_ int, _ time.Duration) (err error) {
282-
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
317+
ctx, cancel := context.WithTimeout(ctx, callTimeout)
283318
defer cancel()
284319
result, err = etcdFunc(*etcdClient, ctx, endpoint)
285320
return err
@@ -300,7 +335,13 @@ func callETCDFuncOnAnyMember[R any](
300335
) (R, error) {
301336
return tracing.WithSpan(ctx, tracer, "CallETCDFuncOnAnyMember",
302337
func(ctx context.Context, span trace.Span) (R, error) {
303-
return callETCDFuncOnMember(ctx, tracerProvider, endpoint, caPool, clientCertificate,
338+
return callETCDFuncOnMember(
339+
ctx,
340+
tracerProvider,
341+
endpoint,
342+
caPool,
343+
clientCertificate,
344+
etcdDefaultCallTimeout,
304345
func(client clientv3.Client, ctx context.Context, _ string) (R, error) {
305346
return etcdFunc(client, ctx)
306347
},

pkg/reconcilers/etcd_cluster/reconciler.go

Lines changed: 67 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ var (
4545
etcdVolumeSizeReCalculatedEvent = "EtcdVolumeSizeRecalculated"
4646
)
4747

48+
const (
49+
etcdDefragmentationFragmentationThreshold = 0.20
50+
etcdDefragmentationCooldown = 1 * time.Hour
51+
)
52+
4853
type EtcdClusterReconciler interface {
4954
ReconcileEtcdCluster(
5055
ctx context.Context,
@@ -179,8 +184,8 @@ func (er *etcdClusterReconciler) ReconcileEtcdCluster(
179184
return "etcd StatefulSet is not ready", nil
180185
}
181186

182-
if err := er.reconcileETCDSpaceUsage(ctx, etcdClient, hostedControlPlane); err != nil {
183-
return "", fmt.Errorf("failed to reconcile etcd space usage: %w", err)
187+
if err := er.reconcileETCDMaintenance(ctx, etcdClient, hostedControlPlane); err != nil {
188+
return "", err
184189
}
185190

186191
if hostedControlPlane.Spec.ETCD.Backup != nil {
@@ -327,18 +332,34 @@ func (er *etcdClusterReconciler) reconcileETCDBackup(
327332
)
328333
}
329334

330-
func (er *etcdClusterReconciler) reconcileETCDSpaceUsage(
335+
func (er *etcdClusterReconciler) reconcileETCDMaintenance(
331336
ctx context.Context,
332337
etcdClient etcd_client.EtcdClient,
333338
hostedControlPlane *v1alpha1.HostedControlPlane,
339+
) error {
340+
statuses, err := etcdClient.GetStatuses(ctx)
341+
if err != nil {
342+
return fmt.Errorf("failed to get etcd statuses: %w", err)
343+
}
344+
345+
if err := er.reconcileETCDSpaceUsage(ctx, statuses, hostedControlPlane); err != nil {
346+
return fmt.Errorf("failed to reconcile etcd space usage: %w", err)
347+
}
348+
349+
if err := er.reconcileETCDDefragmentation(ctx, etcdClient, statuses, hostedControlPlane); err != nil {
350+
return fmt.Errorf("failed to reconcile etcd defragmentation: %w", err)
351+
}
352+
353+
return nil
354+
}
355+
356+
func (er *etcdClusterReconciler) reconcileETCDSpaceUsage(
357+
ctx context.Context,
358+
statuses map[string]*clientv3.StatusResponse,
359+
hostedControlPlane *v1alpha1.HostedControlPlane,
334360
) error {
335361
return tracing.WithSpan1(ctx, er.Tracer, "ReconcileETCDSpaceUsage",
336362
func(ctx context.Context, span trace.Span) (err error) {
337-
statuses, err := etcdClient.GetStatuses(ctx)
338-
if err != nil {
339-
return fmt.Errorf("failed to get etcd member statuses: %w", err)
340-
}
341-
342363
dbSize := slices.Max(slices.Map(slices.Values(statuses),
343364
func(status *clientv3.StatusResponse, _ int) int64 {
344365
return status.DbSize
@@ -356,6 +377,44 @@ func (er *etcdClusterReconciler) reconcileETCDSpaceUsage(
356377
)
357378
}
358379

380+
func (er *etcdClusterReconciler) reconcileETCDDefragmentation(
381+
ctx context.Context,
382+
etcdClient etcd_client.EtcdClient,
383+
statuses map[string]*clientv3.StatusResponse,
384+
hostedControlPlane *v1alpha1.HostedControlPlane,
385+
) error {
386+
return tracing.WithSpan1(ctx, er.Tracer, "ReconcileETCDDefragmentation",
387+
func(ctx context.Context, span trace.Span) error {
388+
if !hostedControlPlane.Status.ETCDLastAttemptedDefragTime.IsZero() &&
389+
time.Since(hostedControlPlane.Status.ETCDLastAttemptedDefragTime.Time) < etcdDefragmentationCooldown {
390+
return nil
391+
}
392+
393+
for _, status := range statuses {
394+
if status.DbSize == 0 || status.DbSizeInUse > status.DbSize ||
395+
float64(
396+
status.DbSize-status.DbSizeInUse,
397+
)/float64(
398+
status.DbSize,
399+
) <= etcdDefragmentationFragmentationThreshold {
400+
continue
401+
}
402+
hostedControlPlane.Status.ETCDLastAttemptedDefragTime = metav1.NewTime(time.Now())
403+
if defragmentationErr := etcdClient.Defragment(ctx); defragmentationErr != nil {
404+
return fmt.Errorf("failed to defragment etcd: %w", defragmentationErr)
405+
}
406+
hostedControlPlane.Status.ETCDLastDefragTime = metav1.NewTime(time.Now())
407+
er.recorder.Normalf(nil, "FragmentationThresholdExceeded", "EtcdDefragmentation",
408+
"Defragmented etcd members due to fragmentation above %.0f%%",
409+
etcdDefragmentationFragmentationThreshold*100,
410+
)
411+
return nil
412+
}
413+
return nil
414+
},
415+
)
416+
}
417+
359418
//+kubebuilder:rbac:groups="",resources=services,verbs=create;patch
360419

361420
func (er *etcdClusterReconciler) reconcileService(

0 commit comments

Comments
 (0)