Skip to content

Commit aa32a2b

Browse files
hjensasdprince
hjensas
authored and
dprince
committed
Leader election lease tunables
Set leader election LeaseDuration, RenewDealine and RetryPeriod via env variables. Doubling the default values from here: https://pkg.go.dev/k8s.io/client-go/tools/leaderelection See also: https://github.com/openshift/enhancements/blob/master/CONVENTIONS.md#high-availability Co-authored-by: Dan Prince <dprince@redhat.com> Jira: OSPRH-16335
1 parent 1ec2176 commit aa32a2b

10 files changed

Lines changed: 83 additions & 4 deletions

File tree

‎bindata/operator/managers.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
{{ $namespace := .OperatorNamespace }}
22
{{ $kubeRbacProxyImage := .KubeRbacProxyImage }}
3+
{{ $leaseDuration := .LeaseDuration }}
4+
{{ $renewDeadline := .RenewDeadline }}
5+
{{ $retryPeriod := .RetryPeriod }}
36
{{ range $operatorName, $operatorImage := .OperatorImages }}
47
apiVersion: apps/v1
58
kind: Deployment
@@ -36,6 +39,12 @@ spec:
3639
{{ else }}
3740
value: 'false'
3841
{{ end }}
42+
- name: LEASE_DURATION
43+
value: '{{ $leaseDuration }}'
44+
- name: RENEW_DEADLINE
45+
value: '{{ $renewDeadline }}'
46+
- name: RETRY_PERIOD
47+
value: '{{ $retryPeriod }}'
3948
image: {{ $operatorImage }}
4049
livenessProbe:
4150
httpGet:

‎bindata/operator/operator.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ spec:
7676
env:
7777
- name: OPENSTACK_RELEASE_VERSION
7878
value: '{{ .OpenstackReleaseVersion }}'
79+
- name: LEASE_DURATION
80+
value: '{{ .LeaseDuration }}'
81+
- name: RENEW_DEADLINE
82+
value: '{{ .RenewDeadline }}'
83+
- name: RETRY_PERIOD
84+
value: '{{ .RetryPeriod }}'
7985
{{ range $envName, $envValue := .OpenStackServiceRelatedImages }}
8086
- name: {{ $envName }}
8187
value: {{ $envValue }}

‎bindata/operator/rabbit.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ spec:
2727
valueFrom:
2828
fieldRef:
2929
fieldPath: metadata.namespace
30+
- name: LEASE_DURATION
31+
value: '{{ .LeaseDuration }}'
32+
- name: RENEW_DEADLINE
33+
value: '{{ .RenewDeadline }}'
34+
- name: RETRY_PERIOD
35+
value: '{{ .RetryPeriod }}'
3036
image: {{ .RabbitmqImage }}
3137
name: operator
3238
ports:

‎cmd/operator/main.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import (
4242

4343
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
4444

45+
"github.com/openstack-k8s-operators/lib-common/modules/common/operator"
4546
operatorv1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/operator/v1beta1"
4647
operatorcontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/operator"
4748
)
@@ -87,7 +88,7 @@ func main() {
8788
c.NextProtos = []string{"http/1.1"}
8889
}
8990

90-
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
91+
options := ctrl.Options{
9192
Scheme: scheme,
9293
Metrics: metricsserver.Options{
9394
BindAddress: metricsAddr,
@@ -111,7 +112,15 @@ func main() {
111112
// if you are doing or is intended to do any operation such as perform cleanups
112113
// after the manager stops then its usage might be unsafe.
113114
// LeaderElectionReleaseOnCancel: true,
114-
})
115+
}
116+
117+
err = operator.SetManagerOptions(&options, setupLog)
118+
if err != nil {
119+
setupLog.Error(err, "unable to set manager options")
120+
os.Exit(1)
121+
}
122+
123+
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
115124
if err != nil {
116125
setupLog.Error(err, "unable to start manager")
117126
os.Exit(1)

‎config/manager/manager.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@ spec:
4444
env:
4545
- name: OPENSTACK_RELEASE_VERSION
4646
value: '{{ .OpenstackReleaseVersion }}'
47+
- name: LEASE_DURATION
48+
value: '{{ .LeaseDuration }}'
49+
- name: RENEW_DEADLINE
50+
value: '{{ .RenewDeadline }}'
51+
- name: RETRY_PERIOD
52+
value: '{{ .RetryPeriod }}'
4753
envCustomImage: replace_me #NOTE: this is used via the Makefile to inject a custom template loop that kustomize won't allow
4854
image: '{{ .OperatorImage }}'
4955
name: manager

‎config/operator/deployment/deployment.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,12 @@ spec:
6969
value: quay.io/openstack-k8s-operators/openstack-operator:latest
7070
- name: ENABLE_WEBHOOKS
7171
value: false
72+
- name: LEASE_DURATION
73+
value: 30
74+
- name: RENEW_DEADLINE
75+
value: 20
76+
- name: RETRY_PERIOD
77+
value: 5
7278
image: controller:latest
7379
name: operator
7480
securityContext:

‎config/operator/managers.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
{{ $namespace := .OperatorNamespace }}
22
{{ $kubeRbacProxyImage := .KubeRbacProxyImage }}
3+
{{ $leaseDuration := .LeaseDuration }}
4+
{{ $renewDeadline := .RenewDeadline }}
5+
{{ $retryPeriod := .RetryPeriod }}
36
{{ range $operatorName, $operatorImage := .OperatorImages }}
47
apiVersion: apps/v1
58
kind: Deployment
@@ -36,6 +39,12 @@ spec:
3639
{{ else }}
3740
value: 'false'
3841
{{ end }}
42+
- name: LEASE_DURATION
43+
value: '{{ $leaseDuration }}'
44+
- name: RENEW_DEADLINE
45+
value: '{{ $renewDeadline }}'
46+
- name: RETRY_PERIOD
47+
value: '{{ $retryPeriod }}'
3948
image: {{ $operatorImage }}
4049
livenessProbe:
4150
httpGet:

‎config/operator/rabbit.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ spec:
2727
valueFrom:
2828
fieldRef:
2929
fieldPath: metadata.namespace
30+
- name: LEASE_DURATION
31+
value: '{{ .LeaseDuration }}'
32+
- name: RENEW_DEADLINE
33+
value: '{{ .RenewDeadline }}'
34+
- name: RETRY_PERIOD
35+
value: '{{ .RetryPeriod }}'
3036
image: {{ .RabbitmqImage }}
3137
name: operator
3238
ports:

‎controllers/operator/openstack_controller.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ var (
7171
operatorImage string
7272
kubeRbacProxyImage string
7373
openstackReleaseVersion string
74+
leaseDuration string
75+
renewDeadline string
76+
retryPeriod string
7477
)
7578

7679
// SetupEnv -
@@ -100,7 +103,14 @@ func SetupEnv() {
100103
operatorImage = envArr[1]
101104
} else if envArr[0] == "OPENSTACK_RELEASE_VERSION" {
102105
openstackReleaseVersion = envArr[1]
106+
} else if envArr[0] == "LEASE_DURATION" {
107+
leaseDuration = envArr[1]
108+
} else if envArr[0] == "RENEW_DEADLINE" {
109+
renewDeadline = envArr[1]
110+
} else if envArr[0] == "RETRY_PERIOD" {
111+
retryPeriod = envArr[1]
103112
}
113+
104114
}
105115
}
106116

@@ -454,6 +464,9 @@ func (r *OpenStackReconciler) applyOperator(ctx context.Context, instance *opera
454464
data.Data["OperatorImage"] = operatorImage
455465
data.Data["KubeRbacProxyImage"] = kubeRbacProxyImage
456466
data.Data["OpenstackReleaseVersion"] = openstackReleaseVersion
467+
data.Data["LeaseDuration"] = leaseDuration
468+
data.Data["RenewDeadline"] = renewDeadline
469+
data.Data["RetryPeriod"] = retryPeriod
457470
data.Data["OpenStackServiceRelatedImages"] = envRelatedOpenStackServiceImages
458471
return r.renderAndApply(ctx, instance, data, "operator", true)
459472
}

‎main.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import (
4444
topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1"
4545
ironicv1 "github.com/openstack-k8s-operators/ironic-operator/api/v1beta1"
4646
keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
47+
"github.com/openstack-k8s-operators/lib-common/modules/common/operator"
4748
manilav1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1"
4849
mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
4950
neutronv1 "github.com/openstack-k8s-operators/neutron-operator/api/v1beta1"
@@ -165,7 +166,7 @@ func main() {
165166
c.NextProtos = []string{"http/1.1"}
166167
}
167168

168-
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
169+
options := ctrl.Options{
169170
Scheme: scheme,
170171
Metrics: metricsserver.Options{
171172
BindAddress: metricsAddr,
@@ -189,7 +190,15 @@ func main() {
189190
// if you are doing or is intended to do any operation such as perform cleanups
190191
// after the manager stops then its usage might be unsafe.
191192
// LeaderElectionReleaseOnCancel: true,
192-
})
193+
}
194+
195+
err = operator.SetManagerOptions(&options, setupLog)
196+
if err != nil {
197+
setupLog.Error(err, "unable to set manager options")
198+
os.Exit(1)
199+
}
200+
201+
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
193202
if err != nil {
194203
setupLog.Error(err, "unable to start manager")
195204
os.Exit(1)

0 commit comments

Comments
 (0)