Skip to content

Commit b83cc86

Browse files
xekxek
committed
Add fernet key rotation cronjob
1 parent 1d792a5 commit b83cc86

6 files changed

Lines changed: 271 additions & 4 deletions

File tree

‎api/bases/keystone.openstack.org_keystoneapis.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,16 @@ spec:
8989
description: EnableSecureRBAC - Enable Consistent and Secure RBAC
9090
policies
9191
type: boolean
92+
fernetMaxActiveKeys:
93+
default: "5"
94+
description: FernetMaxActiveKeys - Maximum number of fernet token
95+
keys after rotation
96+
type: string
97+
fernetRotationSchedule:
98+
default: 1 0 * * *
99+
description: FernetRotationSchedule - Schedule rotate fernet token
100+
keys
101+
type: string
92102
memcachedInstance:
93103
default: memcached
94104
description: Memcached instance name.

‎api/v1beta1/keystoneapi_types.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,16 @@ type KeystoneAPISpecCore struct {
119119
// TrustFlushSuspend - Suspend the cron job to purge trusts
120120
TrustFlushSuspend bool `json:"trustFlushSuspend"`
121121

122+
// +kubebuilder:validation:Optional
123+
// +kubebuilder:default="1 0 * * *"
124+
// FernetRotationSchedule - Schedule rotate fernet token keys
125+
FernetRotationSchedule string `json:"fernetRotationSchedule"`
126+
127+
// +kubebuilder:validation:Optional
128+
// +kubebuilder:default="5"
129+
// FernetMaxActiveKeys - Maximum number of fernet token keys after rotation
130+
FernetMaxActiveKeys string `json:"fernetMaxActiveKeys"`
131+
122132
// +kubebuilder:validation:Optional
123133
// +kubebuilder:default={admin: AdminPassword}
124134
// PasswordSelectors - Selectors to identify the AdminUser password from the Secret
@@ -273,6 +283,16 @@ func (instance KeystoneAPI) RbacResourceName() string {
273283
return "keystone-" + instance.Name
274284
}
275285

286+
// KeystoneAPIFernet - used to create different role for fernet key rotation
287+
type KeystoneAPIFernet struct {
288+
*KeystoneAPI
289+
}
290+
291+
// RbacResourceName - return the name to be used for rbac objects used for fernet key rotation (serviceaccount, role, rolebinding)
292+
func (instance KeystoneAPIFernet) RbacResourceName() string {
293+
return "keystone-fernet-" + instance.Name
294+
}
295+
276296
// SetupDefaults - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks)
277297
func SetupDefaults() {
278298
// Acquire environmental defaults and initialize Keystone defaults with them

‎api/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/keystone.openstack.org_keystoneapis.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,16 @@ spec:
8989
description: EnableSecureRBAC - Enable Consistent and Secure RBAC
9090
policies
9191
type: boolean
92+
fernetMaxActiveKeys:
93+
default: "5"
94+
description: FernetMaxActiveKeys - Maximum number of fernet token
95+
keys after rotation
96+
type: string
97+
fernetRotationSchedule:
98+
default: 1 0 * * *
99+
description: FernetRotationSchedule - Schedule rotate fernet token
100+
keys
101+
type: string
92102
memcachedInstance:
93103
default: memcached
94104
description: Memcached instance name.

‎controllers/keystoneapi_controller.go‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,29 @@ func (r *KeystoneAPIReconciler) reconcileInit(
469469
return rbacResult, nil
470470
}
471471

472+
//
473+
// Service account, role, binding for fernet key rotation
474+
//
475+
fernetRbacRules := []rbacv1.PolicyRule{
476+
{
477+
APIGroups: []string{"security.openshift.io"},
478+
ResourceNames: []string{"anyuid"},
479+
Resources: []string{"securitycontextconstraints"},
480+
Verbs: []string{"use"},
481+
},
482+
{
483+
APIGroups: []string{""},
484+
Resources: []string{"secrets"},
485+
Verbs: []string{"patch"},
486+
},
487+
}
488+
fernetRbacResult, err := common_rbac.ReconcileRbac(ctx, helper, keystonev1.KeystoneAPIFernet{KeystoneAPI: instance}, fernetRbacRules)
489+
if err != nil {
490+
return fernetRbacResult, err
491+
} else if (rbacResult != ctrl.Result{}) {
492+
return fernetRbacResult, nil
493+
}
494+
472495
//
473496
// run keystone db sync
474497
//
@@ -1082,14 +1105,35 @@ func (r *KeystoneAPIReconciler) reconcileNormal(
10821105
}
10831106
}
10841107

1085-
// create CronJob
1108+
// create Trust Flush CronJob
10861109
cronjobDef := keystone.CronJob(instance, serviceLabels, serviceAnnotations)
1087-
cronjob := cronjob.NewCronJob(
1110+
trustflushjob := cronjob.NewCronJob(
10881111
cronjobDef,
10891112
5*time.Second,
10901113
)
10911114

1092-
ctrlResult, err = cronjob.CreateOrPatch(ctx, helper)
1115+
ctrlResult, err = trustflushjob.CreateOrPatch(ctx, helper)
1116+
if err != nil {
1117+
instance.Status.Conditions.Set(condition.FalseCondition(
1118+
condition.CronJobReadyCondition,
1119+
condition.ErrorReason,
1120+
condition.SeverityWarning,
1121+
condition.CronJobReadyErrorMessage,
1122+
err.Error()))
1123+
return ctrlResult, err
1124+
}
1125+
1126+
instance.Status.Conditions.MarkTrue(condition.CronJobReadyCondition, condition.CronJobReadyMessage)
1127+
// create Trust Flush CronJob - end
1128+
1129+
// create Fernet Key Rotation CronJob
1130+
fernetjobDef := keystone.FernetCronJob(instance, serviceLabels, serviceAnnotations)
1131+
fernetjob := cronjob.NewCronJob(
1132+
fernetjobDef,
1133+
5*time.Second,
1134+
)
1135+
1136+
ctrlResult, err = fernetjob.CreateOrPatch(ctx, helper)
10931137
if err != nil {
10941138
instance.Status.Conditions.Set(condition.FalseCondition(
10951139
condition.CronJobReadyCondition,
@@ -1101,7 +1145,7 @@ func (r *KeystoneAPIReconciler) reconcileNormal(
11011145
}
11021146

11031147
instance.Status.Conditions.MarkTrue(condition.CronJobReadyCondition, condition.CronJobReadyMessage)
1104-
// create CronJob - end
1148+
// create Fernet Key Rotation CronJob - end
11051149

11061150
//
11071151
// create OpenStackClient config

‎pkg/keystone/fernet.go‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@ import (
1919
"encoding/base64"
2020

2121
"math/rand"
22+
23+
keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
24+
"github.com/openstack-k8s-operators/lib-common/modules/common/env"
25+
26+
batchv1 "k8s.io/api/batch/v1"
27+
corev1 "k8s.io/api/core/v1"
28+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2229
)
2330

2431
// GenerateFernetKey -
@@ -29,3 +36,159 @@ func GenerateFernetKey() string {
2936
}
3037
return base64.StdEncoding.EncodeToString(data)
3138
}
39+
40+
const (
41+
// FernetRotationCommand -
42+
FernetRotationCommand = `
43+
echo $(date -u) Starting...
44+
case $MAX_ACTIVE_KEYS in
45+
''|*[!0-9]*)
46+
echo "MAX_ACTIVE_KEYS is not a number, exiting."
47+
exit 1
48+
;;
49+
[01])
50+
echo "MAX_ACTIVE_KEYS ($MAX_ACTIVE_KEYS) -lt 2, exiting."
51+
exit 1
52+
esac
53+
54+
cd /var/lib/fernet-keys
55+
mkdir /tmp/keys
56+
for file in FernetKeys[0-9]*;
57+
do
58+
cat "$file" > /tmp/keys/"${file#FernetKeys}"
59+
done
60+
61+
cd /tmp/keys
62+
63+
number_of_keys=$(ls -1 | wc -l)
64+
max_key=$(ls -1 | sort -n | tail -1)
65+
66+
if [ $((number_of_keys - 1)) != $max_key ]; then
67+
echo "Corrupted FernetKeys secret, exiting."
68+
exit 1
69+
fi
70+
71+
mv 0 $((max_key + 1))
72+
dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 > 0
73+
74+
while [ -f "$MAX_ACTIVE_KEYS" ]; do
75+
i=2
76+
while [ -f "$i" ]; do
77+
mv $i $((i-1))
78+
i=$((i+1))
79+
done
80+
done
81+
82+
echo '{"stringData": {' > /tmp/patch_file.json
83+
i=0
84+
while [ -f "$((i+1))" ]; do
85+
echo '"FernetKeys'$i'": "'$(cat $i)'",' >> /tmp/patch_file.json
86+
i=$((i+1))
87+
done
88+
echo '"FernetKeys'$i'": "'$(cat $i)'"' >> /tmp/patch_file.json
89+
echo '}}' >> /tmp/patch_file.json
90+
91+
kubectl patch secret -n $NAMESPACE $SECRET_NAME \
92+
--patch-file=/tmp/patch_file.json
93+
echo $(date -u) $((i+1)) keys rotated.
94+
95+
cd /var/lib/fernet-keys
96+
if [ -f "FernetKeys$MAX_ACTIVE_KEYS" ]; then
97+
echo '[' > /tmp/patch_file.json
98+
i=$((MAX_ACTIVE_KEYS-1))
99+
while [ -f "FernetKeys$((i+1))" ]; do
100+
echo '{"op": "remove", "path": "/data/FernetKeys'$i'"},' \
101+
>> /tmp/patch_file.json
102+
i=$((i+1))
103+
done
104+
echo '{"op": "remove", "path": "/data/FernetKeys'$i'"}' \
105+
>> /tmp/patch_file.json
106+
echo ']' >> /tmp/patch_file.json
107+
108+
kubectl patch secret -n $NAMESPACE $SECRET_NAME \
109+
--type=json --patch-file=/tmp/patch_file.json
110+
echo $(date -u) $MAX_ACTIVE_KEYS through $i keys deleted.
111+
fi
112+
`
113+
)
114+
115+
// FernetCronJob func
116+
func FernetCronJob(
117+
keystoneapiinstance *keystonev1.KeystoneAPI,
118+
labels map[string]string,
119+
annotations map[string]string,
120+
) *batchv1.CronJob {
121+
instance := &keystonev1.KeystoneAPIFernet{KeystoneAPI: keystoneapiinstance}
122+
runAsUser := int64(0)
123+
suspend := false
124+
successfulJobsHistoryLimit := int32(3)
125+
failedJobsHistoryLimit := int32(1)
126+
127+
args := []string{"-c", FernetRotationCommand}
128+
129+
envVars := map[string]env.Setter{}
130+
envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
131+
envVars["SECRET_NAME"] = env.SetValue(ServiceName)
132+
envVars["MAX_ACTIVE_KEYS"] = env.SetValue(
133+
instance.Spec.FernetMaxActiveKeys)
134+
135+
backoffLimit := int32(0)
136+
parallelism := int32(1)
137+
completions := int32(1)
138+
139+
// create Volume and VolumeMounts
140+
volumes := getVolumes(instance.Name)
141+
volumeMounts := getVolumeMounts()
142+
143+
cronjob := &batchv1.CronJob{
144+
ObjectMeta: metav1.ObjectMeta{
145+
Name: ServiceName + "-fernet-cronjob",
146+
Namespace: instance.Namespace,
147+
},
148+
Spec: batchv1.CronJobSpec{
149+
Schedule: instance.Spec.FernetRotationSchedule,
150+
Suspend: &suspend,
151+
ConcurrencyPolicy: batchv1.ForbidConcurrent,
152+
SuccessfulJobsHistoryLimit: &successfulJobsHistoryLimit,
153+
FailedJobsHistoryLimit: &failedJobsHistoryLimit,
154+
JobTemplate: batchv1.JobTemplateSpec{
155+
ObjectMeta: metav1.ObjectMeta{
156+
Annotations: annotations,
157+
Labels: labels,
158+
},
159+
Spec: batchv1.JobSpec{
160+
BackoffLimit: &backoffLimit,
161+
Parallelism: &parallelism,
162+
Completions: &completions,
163+
Template: corev1.PodTemplateSpec{
164+
Spec: corev1.PodSpec{
165+
Containers: []corev1.Container{
166+
{
167+
Name: ServiceName + "-fernet-job",
168+
Image: instance.Spec.ContainerImage,
169+
Command: []string{
170+
"/bin/bash",
171+
},
172+
Args: args,
173+
Env: env.MergeEnvs([]corev1.EnvVar{}, envVars),
174+
VolumeMounts: volumeMounts,
175+
SecurityContext: &corev1.SecurityContext{
176+
RunAsUser: &runAsUser,
177+
},
178+
},
179+
},
180+
Volumes: volumes,
181+
RestartPolicy: corev1.RestartPolicyNever,
182+
ServiceAccountName: instance.RbacResourceName(),
183+
},
184+
},
185+
},
186+
},
187+
},
188+
}
189+
if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 {
190+
cronjob.Spec.JobTemplate.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector
191+
}
192+
193+
return cronjob
194+
}

0 commit comments

Comments
 (0)