diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index dd91e71fb..3079fd5a3 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -316,6 +316,15 @@ type PMMSpec struct { ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"` } +type EncryptionKeySecretSelector struct { + // +kubebuilder:validation:Required + Name string `json:"name"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default:=encryptionKey + Key string `json:"key,omitempty"` +} + type BackupSpec struct { Enabled bool `json:"enabled,omitempty"` SourcePod string `json:"sourcePod,omitempty"` @@ -333,6 +342,19 @@ type BackupSpec struct { // Deprecated: not supported since v0.12.0. Use initContainer instead InitImage string `json:"initImage,omitempty"` InitContainer *InitContainerSpec `json:"initContainer,omitempty"` + + // EncryptionKeySecret is the secret key selector for the backup encryption key. + EncryptionKeySecret *EncryptionKeySecretSelector `json:"encryptionKeySecret,omitempty"` +} + +func (s *BackupSpec) GetEncryptionEnabled(storage *BackupStorageSpec) bool { + if s.EncryptionKeySecret != nil { + return true + } + if storage != nil && storage.EncryptionKeySecret != nil { + return true + } + return false } type BackupSchedule struct { @@ -388,6 +410,11 @@ type BackupStorageSpec struct { RuntimeClassName *string `json:"runtimeClassName,omitempty"` VerifyTLS *bool `json:"verifyTLS,omitempty"` ContainerOptions *BackupContainerOptions `json:"containerOptions,omitempty"` + + // EncryptionKeySecret is the secret key selector for the backup encryption key. + // This takes precedence over the encryption key secret in the backup spec. + // +optional + EncryptionKeySecret *EncryptionKeySecretSelector `json:"encryptionKeySecret,omitempty"` } type BackupContainerOptions struct { @@ -404,6 +431,25 @@ func (b *BackupContainerOptions) GetEnv() []corev1.EnvVar { return util.MergeEnvLists(b.Env, b.Args.env()) } +func (b *BackupContainerOptions) GetArgs() BackupContainerArgs { + if b == nil { + return BackupContainerArgs{} + } + return b.Args +} + +func (args BackupContainerArgs) GetXtrabackupFlagValue(flag string) string { + matchPrefix := flag + "=" + for _, arg := range args.Xtrabackup { + if !strings.HasPrefix(arg, matchPrefix) { + continue + } + + return strings.TrimPrefix(arg, matchPrefix) + } + return "" +} + type BackupContainerArgs struct { Xtrabackup []string `json:"xtrabackup,omitempty"` Xbcloud []string `json:"xbcloud,omitempty"` diff --git a/api/v1/perconaservermysql_types_test.go b/api/v1/perconaservermysql_types_test.go index a5c8b1b3d..8564c91e7 100644 --- a/api/v1/perconaservermysql_types_test.go +++ b/api/v1/perconaservermysql_types_test.go @@ -752,3 +752,59 @@ func TestBackupStorageSpecEquals(t *testing.T) { }) } } + +func TestGetXtrabackupFlagValue(t *testing.T) { + testCases := []struct { + containerOptions BackupContainerOptions + flag string + want string + }{ + { + containerOptions: BackupContainerOptions{ + Args: BackupContainerArgs{ + Xtrabackup: []string{"--compress=lz4"}, + }, + }, + flag: "--compress", + want: "lz4", + }, + { + containerOptions: BackupContainerOptions{ + Args: BackupContainerArgs{ + Xtrabackup: []string{"--encrypt=AES256"}, + }, + }, + flag: "--encrypt", + want: "AES256", + }, + { + containerOptions: BackupContainerOptions{ + Args: BackupContainerArgs{ + Xtrabackup: []string{"--encrypt=AES256", "--encrypt-key-file=encryption-key"}, + }, + }, + flag: "--encrypt", + want: "AES256", + }, + { + containerOptions: BackupContainerOptions{ + Args: BackupContainerArgs{ + Xtrabackup: []string{"--encrypt=AES256", "--encrypt-key-file=encryption-key"}, + }, + }, + flag: "--encrypt-key-file", + want: "encryption-key", + }, + { + containerOptions: BackupContainerOptions{}, + flag: "--encrypt", + }, + } + + for _, tc := range testCases { + t.Run(tc.flag, func(t *testing.T) { + got := tc.containerOptions.GetArgs().GetXtrabackupFlagValue(tc.flag) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 222bdb2bb..5528b87c8 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -142,6 +142,11 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = new(InitContainerSpec) (*in).DeepCopyInto(*out) } + if in.EncryptionKeySecret != nil { + in, out := &in.EncryptionKeySecret, &out.EncryptionKeySecret + *out = new(EncryptionKeySecretSelector) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec. @@ -288,6 +293,11 @@ func (in *BackupStorageSpec) DeepCopyInto(out *BackupStorageSpec) { *out = new(BackupContainerOptions) (*in).DeepCopyInto(*out) } + if in.EncryptionKeySecret != nil { + in, out := &in.EncryptionKeySecret, &out.EncryptionKeySecret + *out = new(EncryptionKeySecretSelector) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStorageSpec. @@ -498,6 +508,21 @@ func (in *CreateReplicaClusterOptions) DeepCopy() *CreateReplicaClusterOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EncryptionKeySecretSelector) DeepCopyInto(out *EncryptionKeySecretSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionKeySecretSelector. +func (in *EncryptionKeySecretSelector) DeepCopy() *EncryptionKeySecretSelector { + if in == nil { + return nil + } + out := new(EncryptionKeySecretSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HAProxySpec) DeepCopyInto(out *HAProxySpec) { *out = *in diff --git a/build/run-backup.sh b/build/run-backup.sh index d33687095..c92009d65 100755 --- a/build/run-backup.sh +++ b/build/run-backup.sh @@ -14,6 +14,8 @@ request_data() { "containerOptions": ${CONTAINER_OPTIONS}, "verifyTLS": $(json_escape "${VERIFY_TLS}"), "incrementalLsn": "$(json_escape "${INCREMENTAL_LSN}")", + "encryptionKeyFile": "$(json_escape "${ENCRYPTION_KEY_FILE}")", + "encryptionKeyVersion": "$(json_escape "${ENCRYPTION_KEY_FILE_VERSION}")", "s3": { "bucket": "$(json_escape "${S3_BUCKET}")", "endpointUrl": "$(json_escape "${AWS_ENDPOINT}")", @@ -32,6 +34,8 @@ request_data() { "type": "$(json_escape "${STORAGE_TYPE}")", "containerOptions": ${CONTAINER_OPTIONS}, "incrementalLsn": "$(json_escape "${INCREMENTAL_LSN}")", + "encryptionKeyFile": "$(json_escape "${ENCRYPTION_KEY_FILE}")", + "encryptionKeyVersion": "$(json_escape "${ENCRYPTION_KEY_FILE_VERSION}")", "gcs": { "bucket": "$(json_escape "${GCS_BUCKET}")", "endpointUrl": "$(json_escape "${GCS_ENDPOINT}")", @@ -50,6 +54,8 @@ request_data() { "type": "$(json_escape "${STORAGE_TYPE}")", "containerOptions": ${CONTAINER_OPTIONS}, "incrementalLsn": "$(json_escape "${INCREMENTAL_LSN}")", + "encryptionKeyFile": "$(json_escape "${ENCRYPTION_KEY_FILE}")", + "encryptionKeyVersion": "$(json_escape "${ENCRYPTION_KEY_FILE_VERSION}")", "azure": { "containerName": "$(json_escape "${AZURE_CONTAINER_NAME}")", "storageAccount": "$(json_escape "${AZURE_STORAGE_ACCOUNT}")", diff --git a/build/run-restore.sh b/build/run-restore.sh index 940e6b54b..acf14aa53 100755 --- a/build/run-restore.sh +++ b/build/run-restore.sh @@ -3,6 +3,8 @@ set -e set -o xtrace +encryption_key_file="/etc/mysql/encryption-keys/encryption-key" + XTRABACKUP_VERSION=$(xtrabackup --version 2>&1 | awk '/^xtrabackup version/{print $3}' | awk -F'.' '{print $1"."$2}') DATADIR=${DATADIR:-/var/lib/mysql} PARALLEL=$(grep -c processor /proc/cpuinfo) @@ -41,6 +43,15 @@ extract() { xbstream -xv -C "${targetdir}" --parallel="${PARALLEL}" ${XBSTREAM_EXTRA_ARGS} } +decrypt() { + local targetdir=$1 + if [ -n "${ENCRYPTION_ALGORITHM}" ]; then + # shellcheck disable=SC2086 + xtrabackup --decrypt=${ENCRYPTION_ALGORITHM} --encrypt-key-file=${encryption_key_file} --target-dir="${targetdir}" --parallel="${PARALLEL}" ${XB_EXTRA_ARGS} + find "${targetdir}" -name '*.xbcrypt' -delete + fi +} + get_keyring_arg() { local keyring="" if [[ -f ${KEYRING_VAULT_PATH} ]]; then @@ -63,7 +74,9 @@ restore_full() { rm -rf "${DATADIR:?}"/* tmpdir=$(mktemp --directory "${DATADIR}/${RESTORE_NAME}_XXXX") + echo "Downloading backup: ${BACKUP_DEST}" download "${BACKUP_DEST}" | extract "${tmpdir}" + decrypt "${tmpdir}" # shellcheck disable=SC2086 xtrabackup --decompress --remove-original --parallel="${PARALLEL}" --target-dir="${tmpdir}" ${XB_EXTRA_ARGS} @@ -93,6 +106,7 @@ restore_incremental() { # Download and extract the base (full) backup echo "Downloading base backup: ${BACKUP_DEST}" download "${BACKUP_DEST}" | extract "${basedir}" + decrypt "${basedir}" # shellcheck disable=SC2086 xtrabackup --decompress --remove-original --parallel="${PARALLEL}" --target-dir="${basedir}" ${XB_EXTRA_ARGS} @@ -117,6 +131,7 @@ restore_incremental() { incrdir=$(mktemp --directory "${DATADIR}/${RESTORE_NAME}_incr${count}_XXXX") download "${incr_dest}" | extract "${incrdir}" + decrypt "${incrdir}" # shellcheck disable=SC2086 xtrabackup --decompress --remove-original --parallel="${PARALLEL}" --target-dir="${incrdir}" ${XB_EXTRA_ARGS} diff --git a/cmd/example-gen/main.go b/cmd/example-gen/main.go index 30ef3013c..593b37d61 100644 --- a/cmd/example-gen/main.go +++ b/cmd/example-gen/main.go @@ -213,6 +213,10 @@ func printRestore() error { NodeSelector: defaults.NodeSelector, VerifyTLS: defaults.VerifyTLS, RuntimeClassName: defaults.RuntimeClassName, + EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{ + Name: "my-encryption-key-secret", + Key: "encryptionKey", + }, }, }, }, diff --git a/cmd/example-gen/pkg/defaults/manual.go b/cmd/example-gen/pkg/defaults/manual.go index 66eb65ec5..a272c1cae 100644 --- a/cmd/example-gen/pkg/defaults/manual.go +++ b/cmd/example-gen/pkg/defaults/manual.go @@ -147,6 +147,11 @@ func backupDefaults(spec *apiv1.BackupSpec) { LogLevel: "info", }, } + + spec.EncryptionKeySecret = &apiv1.EncryptionKeySecretSelector{ + Name: "my-encryption-key-secret", + Key: "encryptionKey", + } podSpecDefaults(&spec.PiTR.BinlogServer.PodSpec, ImageBinlogServer, corev1.ResourceRequirements{}, "", 30, nil, nil) spec.PiTR.BinlogServer.Size = 1 spec.SourcePod = SourcePod @@ -176,6 +181,10 @@ func backupDefaults(spec *apiv1.BackupSpec) { EndpointURL: "https://accountName.blob.core.windows.net", StorageClass: "Cool", }, + EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{ + Name: "my-azure-encryption-key-secret", + Key: "encryptionKey", + }, }, "gcp-cs": { Type: apiv1.BackupStorageGCS, @@ -185,6 +194,10 @@ func backupDefaults(spec *apiv1.BackupSpec) { CredentialsSecret: "SECRET-NAME", EndpointURL: "https://storage.googleapis.com", }, + EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{ + Name: "my-gcs-encryption-key-secret", + Key: "encryptionKey", + }, }, "s3-us-west": { Type: apiv1.BackupStorageS3, @@ -206,6 +219,10 @@ func backupDefaults(spec *apiv1.BackupSpec) { SchedulerName: SchedulerName, VerifyTLS: VerifyTLS, NodeSelector: NodeSelector, + EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{ + Name: "my-s3-encryption-key-secret", + Key: "encryptionKey", + }, }, } } diff --git a/cmd/example-gen/scripts/lib/ps.sh b/cmd/example-gen/scripts/lib/ps.sh index 34414cc8b..8c0b60605 100644 --- a/cmd/example-gen/scripts/lib/ps.sh +++ b/cmd/example-gen/scripts/lib/ps.sh @@ -220,6 +220,7 @@ del_fields_to_comment() { | yq "del(.spec.backup.containerSecurityContext)" \ | yq "del(.spec.backup.resources)" \ | yq "del(.spec.backup.serviceAccountName)" \ + | yq "del(.spec.backup.encryptionKeySecret)" \ | yq "del(.spec.backup.storages.azure-blob)" \ | yq "del(.spec.backup.storages.gcp-cs)" \ | yq "del(.spec.backup.storages.s3-us-west.resources)" \ @@ -238,6 +239,7 @@ del_fields_to_comment() { | yq "del(.spec.backup.storages.s3-us-west.s3.endpointUrl)" \ | yq "del(.spec.backup.storages.s3-us-west.schedulerName)" \ | yq "del(.spec.backup.storages.s3-us-west.runtimeClassName)" \ + | yq "del(.spec.backup.storages.s3-us-west.encryptionKeySecret)" \ | yq "del(.spec.toolkit.imagePullSecrets)" \ | yq "del(.spec.toolkit.env)" \ | yq "del(.spec.toolkit.envFrom)" \ diff --git a/cmd/sidecar/handler/backup/create.go b/cmd/sidecar/handler/backup/create.go index 0bc932fab..1feb8493f 100644 --- a/cmd/sidecar/handler/backup/create.go +++ b/cmd/sidecar/handler/backup/create.go @@ -10,7 +10,9 @@ import ( "os/exec" "path/filepath" "strings" + "time" + "github.com/go-logr/logr" "github.com/pkg/errors" "golang.org/x/sync/errgroup" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -62,6 +64,15 @@ func (h *Handler) createBackupHandler(w http.ResponseWriter, req *http.Request) return } + // Wait for kubelet to propagate the encryption key file to the pod. + if backupConf.EncryptionKeyFile != "" { + if err := awaitEncryptionKeyFile(req.Context(), log, backupConf.EncryptionKeyFile, backupConf.EncryptionKeyVersion); err != nil { + log.Error(err, "failed to await encryption key file") + http.Error(w, "backup failed", http.StatusInternalServerError) + return + } + } + h.status.setBackupConfig(backupConf) defer func() { h.status.removeBackupConfig() @@ -93,7 +104,6 @@ func (h *Handler) createBackupHandler(w http.ResponseWriter, req *http.Request) return } g, gCtx := errgroup.WithContext(req.Context()) - xtrabackup := exec.CommandContext(gCtx, "xtrabackup", xtrabackupArgs(string(backupUser), backupPass, &backupConf)...) xtrabackup.Env = envs(backupConf) @@ -213,12 +223,19 @@ func xtrabackupArgs(user, pass string, conf *xb.BackupConfig) []string { if _, err := os.Stat(mysql.CustomMyCnfPath); err == nil { args = append([]string{"--defaults-extra-file=" + mysql.CustomMyCnfPath}, args...) } + if conf != nil && conf.EncryptionKeyFile != "" { + args = append(args, fmt.Sprintf("--encrypt-key-file=%s", conf.EncryptionKeyFile)) + if conf.ContainerOptions.GetArgs().GetXtrabackupFlagValue("--encrypt") == "" { + args = append(args, "--encrypt=AES256") + } + } if conf != nil && conf.ContainerOptions != nil { args = append(args, conf.ContainerOptions.Args.Xtrabackup...) } if conf != nil && conf.IncrementalLsn != "" { args = append(args, fmt.Sprintf("--incremental-lsn=%s", conf.IncrementalLsn)) } + return args } @@ -296,3 +313,50 @@ func getSecret(username apiv1.SystemUser) (string, error) { return strings.TrimSpace(string(sBytes)), nil } + +func versionFileFor(keyFile string) string { + return filepath.Join(filepath.Dir(keyFile), naming.InternalEncryptionKeyVersionFileName) +} + +func readVersionFile(path string) (string, error) { + sBytes, err := os.ReadFile(path) + if err != nil { + return "", errors.Wrapf(err, "read %s", path) + } + return strings.TrimSpace(string(sBytes)), nil +} + +const defaultKubeletSecretSyncTimeout = 5 * time.Minute + +// awaitEncryptionKeyFile waits for the encryption key file to be created and synced from the Secret. +func awaitEncryptionKeyFile(ctx context.Context, log logr.Logger, file, desiredVersion string) error { + pCtx, cancel := context.WithTimeout(ctx, defaultKubeletSecretSyncTimeout) + defer cancel() + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + // Wait for the file to exist. + if _, err := os.Stat(file); err == nil { + // If the file exists, wait for it to be synced from the Secret. + observedVersion, err := readVersionFile(versionFileFor(file)) + if err != nil { + return errors.Wrap(err, "read version file") + } + if observedVersion == desiredVersion { + return nil + } + } else if !os.IsNotExist(err) { + return errors.Wrap(err, "stat encryption key file") + } + + select { + case <-pCtx.Done(): + return errors.Wrap(pCtx.Err(), "context done") + + case <-ticker.C: + log.Info("waiting for encryption key file", "file", file) + } + } +} diff --git a/config/crd/bases/ps.percona.com_perconaservermysqlbackups.yaml b/config/crd/bases/ps.percona.com_perconaservermysqlbackups.yaml index 451eb738b..439761fc3 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqlbackups.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqlbackups.yaml @@ -859,6 +859,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: diff --git a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml index a95d0b040..bd1af11f4 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml @@ -724,6 +724,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 24e4fe204..df66f06a8 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -132,6 +132,16 @@ spec: type: object enabled: type: boolean + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object image: type: string imagePullPolicy: @@ -2243,6 +2253,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: diff --git a/deploy/backup/restore.yaml b/deploy/backup/restore.yaml index b8dc9f1bb..079f93ccb 100644 --- a/deploy/backup/restore.yaml +++ b/deploy/backup/restore.yaml @@ -70,6 +70,9 @@ spec: # privileged: false # runAsGroup: 1001 # runAsUser: 1001 +# encryptionKeySecret: +# key: encryptionKey +# name: my-encryption-key-secret # labels: # rack: rack-22 # nodeSelector: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 07b60799b..4e0559bb0 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -863,6 +863,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: @@ -2161,6 +2171,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: @@ -4066,6 +4086,16 @@ spec: type: object enabled: type: boolean + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object image: type: string imagePullPolicy: @@ -6177,6 +6207,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: diff --git a/deploy/cr.yaml b/deploy/cr.yaml index c6e3a9e20..6686f28e1 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -811,6 +811,9 @@ spec: # privileged: false # runAsGroup: 1001 # runAsUser: 1001 +# encryptionKeySecret: +# key: encryptionKey +# name: my-azure-encryption-key-secret # podSecurityContext: # fsGroup: 1001 # supplementalGroups: @@ -854,6 +857,9 @@ spec: # privileged: false # runAsGroup: 1001 # runAsUser: 1001 +# encryptionKeySecret: +# key: encryptionKey +# name: my-gcs-encryption-key-secret # gcs: # bucket: BUCKET-NAME # credentialsSecret: SECRET-NAME @@ -914,6 +920,9 @@ spec: # privileged: false # runAsGroup: 1001 # runAsUser: 1001 +# encryptionKeySecret: +# key: encryptionKey +# name: my-s3-encryption-key-secret # labels: # backupWorker: "True" # nodeSelector: @@ -966,6 +975,9 @@ spec: # requests: # storage: 2Gi # storageClassName: standard +# encryptionKeySecret: +# key: encryptionKey +# name: my-encryption-key-secret toolkit: image: perconalab/percona-server-mysql-operator:main-toolkit imagePullPolicy: Always diff --git a/deploy/crd.yaml b/deploy/crd.yaml index d80492029..e5ec0d704 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -863,6 +863,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: @@ -2161,6 +2171,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: @@ -4066,6 +4086,16 @@ spec: type: object enabled: type: boolean + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object image: type: string imagePullPolicy: @@ -6177,6 +6207,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index c7a4ae54b..7fae52f3d 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -863,6 +863,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: @@ -2161,6 +2171,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: @@ -4066,6 +4086,16 @@ spec: type: object enabled: type: boolean + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object image: type: string imagePullPolicy: @@ -6177,6 +6207,16 @@ spec: type: string type: object type: object + encryptionKeySecret: + properties: + key: + default: encryptionKey + type: string + name: + type: string + required: + - name + type: object gcs: properties: bucket: diff --git a/e2e-tests/tests/auto-config/01-assert.yaml b/e2e-tests/tests/auto-config/01-assert.yaml index af7d723ff..3cdfb9cb3 100644 --- a/e2e-tests/tests/auto-config/01-assert.yaml +++ b/e2e-tests/tests/auto-config/01-assert.yaml @@ -41,6 +41,8 @@ spec: name: vault-keyring-secret - mountPath: /etc/mysql/config name: config + - mountPath: /etc/mysql/encryption-keys + name: backup-encryption-keys - name: pt-heartbeat volumeMounts: - mountPath: /opt/percona @@ -91,6 +93,11 @@ spec: defaultMode: 420 optional: true secretName: auto-config-vault + - name: backup-encryption-keys + secret: + defaultMode: 420 + optional: true + secretName: internal-encryption-keys-auto-config status: observedGeneration: 1 replicas: 3 diff --git a/e2e-tests/tests/config-router/01-assert.yaml b/e2e-tests/tests/config-router/01-assert.yaml index a6f0a0707..6207a4444 100644 --- a/e2e-tests/tests/config-router/01-assert.yaml +++ b/e2e-tests/tests/config-router/01-assert.yaml @@ -41,6 +41,8 @@ spec: name: vault-keyring-secret - mountPath: /etc/mysql/config name: config + - mountPath: /etc/mysql/encryption-keys + name: backup-encryption-keys volumes: - emptyDir: {} name: bin @@ -83,6 +85,11 @@ spec: defaultMode: 420 optional: true secretName: config-router-vault + - name: backup-encryption-keys + secret: + defaultMode: 420 + optional: true + secretName: internal-encryption-keys-config-router status: observedGeneration: 1 replicas: 3 diff --git a/e2e-tests/tests/config/01-assert.yaml b/e2e-tests/tests/config/01-assert.yaml index b1baf1963..5c84b55c1 100644 --- a/e2e-tests/tests/config/01-assert.yaml +++ b/e2e-tests/tests/config/01-assert.yaml @@ -41,6 +41,8 @@ spec: name: vault-keyring-secret - mountPath: /etc/mysql/config name: config + - mountPath: /etc/mysql/encryption-keys + name: backup-encryption-keys - name: pt-heartbeat volumeMounts: - mountPath: /opt/percona @@ -91,6 +93,11 @@ spec: defaultMode: 420 optional: true secretName: config-vault + - name: backup-encryption-keys + secret: + defaultMode: 420 + optional: true + secretName: internal-encryption-keys-config status: observedGeneration: 1 replicas: 3 diff --git a/e2e-tests/tests/demand-backup-incremental/96-delete-backups.yaml b/e2e-tests/tests/demand-backup-incremental/96-delete-backups.yaml index 9deccf58c..6d6c303dd 100644 --- a/e2e-tests/tests/demand-backup-incremental/96-delete-backups.yaml +++ b/e2e-tests/tests/demand-backup-incremental/96-delete-backups.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 +timeout: 90 delete: # Deleting base backup should clear the entire chain - apiVersion: ps.percona.com/v1 diff --git a/e2e-tests/tests/gr-demand-backup-incremental/33-create-encryption-key.yaml b/e2e-tests/tests/gr-demand-backup-incremental/33-create-encryption-key.yaml new file mode 100644 index 000000000..65cf243d5 --- /dev/null +++ b/e2e-tests/tests/gr-demand-backup-incremental/33-create-encryption-key.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + key=$(openssl rand -base64 24) + secretName="test-encryption-key" + + kubectl create secret generic "${secretName}" --from-literal=encryptionKey="${key}" -n "${NAMESPACE}" + + patch=$(cat < + finalizers: + - percona.com/delete-backup +spec: + clusterName: gr-demand-backup-incremental + storageName: + type: full + containerOptions: + args: + xtrabackup: + - "--compress" \ No newline at end of file diff --git a/e2e-tests/tests/gr-demand-backup-incremental/conf/backup/backup-encrypted-incremental.yaml b/e2e-tests/tests/gr-demand-backup-incremental/conf/backup/backup-encrypted-incremental.yaml new file mode 100644 index 000000000..c8db68f6e --- /dev/null +++ b/e2e-tests/tests/gr-demand-backup-incremental/conf/backup/backup-encrypted-incremental.yaml @@ -0,0 +1,14 @@ +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQLBackup +metadata: + name: + finalizers: + - percona.com/delete-backup +spec: + clusterName: gr-demand-backup-incremental + storageName: + type: incremental + containerOptions: + args: + xtrabackup: + - "--compress" \ No newline at end of file diff --git a/e2e-tests/tests/gr-demand-backup-incremental/conf/backup/restore-backup-source-encrypted.yaml b/e2e-tests/tests/gr-demand-backup-incremental/conf/backup/restore-backup-source-encrypted.yaml new file mode 100644 index 000000000..1ca570bc0 --- /dev/null +++ b/e2e-tests/tests/gr-demand-backup-incremental/conf/backup/restore-backup-source-encrypted.yaml @@ -0,0 +1,18 @@ +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQLRestore +metadata: + name: +spec: + clusterName: gr-demand-backup-incremental + backupSource: + destination: + storage: + encryptionKeySecret: + name: test-encryption-key + key: encryptionKey + type: s3 + s3: + bucket: operator-testing + credentialsSecret: minio-secret + endpointUrl: http://minio-service.:9000 + region: us-east-1 diff --git a/e2e-tests/tests/limits/01-assert.yaml b/e2e-tests/tests/limits/01-assert.yaml index 3a893d760..3795cc9f8 100644 --- a/e2e-tests/tests/limits/01-assert.yaml +++ b/e2e-tests/tests/limits/01-assert.yaml @@ -163,6 +163,8 @@ spec: name: vault-keyring-secret - mountPath: /etc/mysql/config name: config + - mountPath: /etc/mysql/encryption-keys + name: backup-encryption-keys - command: - /opt/percona/heartbeat-entrypoint.sh imagePullPolicy: Always @@ -237,6 +239,11 @@ spec: defaultMode: 420 optional: true secretName: limits-no-limits-vault + - name: backup-encryption-keys + secret: + defaultMode: 420 + optional: true + secretName: internal-encryption-keys-limits-no-limits updateStrategy: type: OnDelete volumeClaimTemplates: diff --git a/e2e-tests/tests/limits/03-assert.yaml b/e2e-tests/tests/limits/03-assert.yaml index 585176c57..716003548 100644 --- a/e2e-tests/tests/limits/03-assert.yaml +++ b/e2e-tests/tests/limits/03-assert.yaml @@ -163,6 +163,8 @@ spec: name: vault-keyring-secret - mountPath: /etc/mysql/config name: config + - mountPath: /etc/mysql/encryption-keys + name: backup-encryption-keys - command: - /opt/percona/heartbeat-entrypoint.sh imagePullPolicy: Always @@ -237,6 +239,11 @@ spec: defaultMode: 420 optional: true secretName: limits-no-requests-vault + - name: backup-encryption-keys + secret: + defaultMode: 420 + optional: true + secretName: internal-encryption-keys-limits-no-requests updateStrategy: type: OnDelete volumeClaimTemplates: diff --git a/e2e-tests/tests/limits/05-assert.yaml b/e2e-tests/tests/limits/05-assert.yaml index 249fd4ef8..500c150e1 100644 --- a/e2e-tests/tests/limits/05-assert.yaml +++ b/e2e-tests/tests/limits/05-assert.yaml @@ -161,6 +161,8 @@ spec: name: vault-keyring-secret - mountPath: /etc/mysql/config name: config + - mountPath: /etc/mysql/encryption-keys + name: backup-encryption-keys - command: - /opt/percona/heartbeat-entrypoint.sh imagePullPolicy: Always @@ -235,6 +237,11 @@ spec: defaultMode: 420 optional: true secretName: limits-no-resources-vault + - name: backup-encryption-keys + secret: + defaultMode: 420 + optional: true + secretName: internal-encryption-keys-limits-no-resources updateStrategy: type: OnDelete volumeClaimTemplates: diff --git a/e2e-tests/tests/sidecars/01-assert.yaml b/e2e-tests/tests/sidecars/01-assert.yaml index 76a6bb709..04d098cf1 100644 --- a/e2e-tests/tests/sidecars/01-assert.yaml +++ b/e2e-tests/tests/sidecars/01-assert.yaml @@ -43,6 +43,8 @@ spec: name: vault-keyring-secret - mountPath: /etc/mysql/config name: config + - mountPath: /etc/mysql/encryption-keys + name: backup-encryption-keys - name: pt-heartbeat volumeMounts: - mountPath: /opt/percona @@ -114,6 +116,11 @@ spec: defaultMode: 420 optional: true secretName: sidecars-vault + - name: backup-encryption-keys + secret: + defaultMode: 420 + optional: true + secretName: internal-encryption-keys-sidecars - emptyDir: medium: Memory name: empty-vol diff --git a/pkg/controller/ps/backup_test.go b/pkg/controller/ps/backup_test.go index 8cfbf11b8..3073ad26a 100644 --- a/pkg/controller/ps/backup_test.go +++ b/pkg/controller/ps/backup_test.go @@ -6,9 +6,17 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/naming" + "github.com/percona/percona-server-mysql-operator/pkg/version" ) func TestGenerateBackupName(t *testing.T) { @@ -136,3 +144,82 @@ func TestGenerateBackupName(t *testing.T) { }) } } + +func TestReconcileInternalEncryptionKeySecret(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, apiv1.AddToScheme(scheme)) + + cr := &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "test-ns", + UID: types.UID("cluster1-uid"), + }, + Spec: apiv1.PerconaServerMySQLSpec{ + CRVersion: version.Version(), + Backup: &apiv1.BackupSpec{ + EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{ + Name: "cluster-key", + Key: "encryptionKey", + }, + Storages: map[string]*apiv1.BackupStorageSpec{ + "s3": { + EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{ + Name: "storage-key", + Key: "custom-key", + }, + }, + "without-key": {}, + "nil": nil, + }, + }, + }, + } + clusterKeySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-key", + Namespace: cr.Namespace, + }, + Data: map[string][]byte{ + "encryptionKey": []byte("cluster-secret-key"), + }, + } + storageKeySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "storage-key", + Namespace: cr.Namespace, + }, + Data: map[string][]byte{ + "custom-key": []byte("storage-secret-key"), + }, + } + + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(cr, clusterKeySecret, storageKeySecret). + Build() + + r := &PerconaServerMySQLReconciler{ + Client: cl, + Scheme: scheme, + } + + require.NoError(t, r.reconcileInternalEncryptionKeySecret(t.Context(), cr)) + + internalSecret := &corev1.Secret{} + require.NoError(t, r.Get(t.Context(), client.ObjectKey{ + Name: naming.EncryptionKeyInternalSecretName(cr.Name), + Namespace: cr.Namespace, + }, internalSecret)) + + assert.Equal(t, []byte("cluster-secret-key"), internalSecret.Data[naming.InternalEncryptionKeyFileName(cr.Name, "")]) + assert.Equal(t, []byte("storage-secret-key"), internalSecret.Data[naming.InternalEncryptionKeyFileName(cr.Name, "s3")]) + assert.NotEmpty(t, internalSecret.Data[naming.InternalEncryptionKeyVersionFileName]) + assert.Len(t, internalSecret.Data, 3) + require.Len(t, internalSecret.OwnerReferences, 1) + assert.Equal(t, cr.Name, internalSecret.OwnerReferences[0].Name) + assert.Equal(t, cr.UID, internalSecret.OwnerReferences[0].UID) + require.NotNil(t, internalSecret.OwnerReferences[0].Controller) + assert.True(t, *internalSecret.OwnerReferences[0].Controller) +} diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index bf3b7e937..bb2ec6fbd 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -45,6 +45,7 @@ import ( k8sretry "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -95,15 +96,18 @@ func (r *PerconaServerMySQLReconciler) SetupWithManager(mgr ctrl.Manager) error return ctrl.NewControllerManagedBy(mgr). For(&apiv1.PerconaServerMySQL{}). Watches(&corev1.Secret{}, enqueueClusterFromSecretsName(r.Client)). + Watches(&corev1.Secret{}, enqueueClusterFromEncryptionKeySecret(r.Client)). Named("ps-controller"). Complete(r) } const ( - fieldSecretsName = ".spec.secretsName" + fieldSecretsName = ".spec.secretsName" + fieldEncryptionKeySecretName = ".spec.backup.encryptionKeySecret.name" ) func setupFieldIndexers(mgr ctrl.Manager) error { + // Index cluster secrets field if err := mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.PerconaServerMySQL{}, fieldSecretsName, func(o client.Object) []string { cluster, ok := o.(*apiv1.PerconaServerMySQL) if !ok { @@ -117,6 +121,34 @@ func setupFieldIndexers(mgr ctrl.Manager) error { }); err != nil { return errors.Wrapf(err, "index field %s", fieldSecretsName) } + + // Index cluster backup encryption key secrets field + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &apiv1.PerconaServerMySQL{}, + fieldEncryptionKeySecretName, + func(rawObj client.Object) []string { + cluster := rawObj.(*apiv1.PerconaServerMySQL) + if cluster.Spec.Backup == nil { + return nil + } + result := []string{} + + if cluster.Spec.Backup.EncryptionKeySecret != nil { + result = append(result, cluster.Spec.Backup.EncryptionKeySecret.Name) + } + + for _, storage := range cluster.Spec.Backup.Storages { + if storage == nil || storage.EncryptionKeySecret == nil { + continue + } + result = append(result, storage.EncryptionKeySecret.Name) + } + return result + }, + ); err != nil { + return errors.Wrapf(err, "unable to index field %s", fieldEncryptionKeySecretName) + } return nil } @@ -148,6 +180,35 @@ func enqueueClusterFromSecretsName(c client.Client) handler.EventHandler { }) } +// enqueueClusterFromEncryptionKeySecret enqueues a request for all clusters that have an encryption key secret set to the given secret. +func enqueueClusterFromEncryptionKeySecret(c client.Client) handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { + secret, ok := o.(*corev1.Secret) + if !ok { + return nil + } + clusters := &apiv1.PerconaServerMySQLList{} + log := logf.FromContext(ctx) + if err := c.List(ctx, clusters, client.InNamespace(secret.Namespace), client.MatchingFields{ + fieldEncryptionKeySecretName: secret.Name, + }); err != nil { + log.Error(err, "failed to list clusters") + return nil + } + + var requests []reconcile.Request + for _, cluster := range clusters.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: cluster.Name, + Namespace: cluster.Namespace, + }, + }) + } + return requests + }) +} + // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. // TODO(user): Modify the Reconcile function to compare the state specified by @@ -414,7 +475,7 @@ func (r *PerconaServerMySQLReconciler) deleteCerts(ctx context.Context, cr *apiv } for _, secretName := range secretNames { secret := &corev1.Secret{} - err := r.Client.Get(ctx, types.NamespacedName{ + err := r.Get(ctx, types.NamespacedName{ Namespace: cr.Namespace, Name: secretName, }, secret) @@ -506,6 +567,9 @@ func (r *PerconaServerMySQLReconciler) doReconcile( if err := r.ensureTLSSecret(ctx, cr); err != nil { return errors.Wrap(err, "TLS secret") } + if err := r.reconcileInternalEncryptionKeySecret(ctx, cr); err != nil { + return errors.Wrap(err, "internal encryption key secret") + } if err := r.reconcileServices(ctx, cr); err != nil { return errors.Wrap(err, "services") } @@ -1710,3 +1774,121 @@ func getPodIndexFromHostname(hostname string) (int, error) { return idx, nil } + +func readEncryptionKey(ctx context.Context, cl client.Client, sel apiv1.EncryptionKeySecretSelector, namespace string) ([]byte, error) { + key := sel.Key + if key == "" { + key = "encryptionKey" + } + + secret := &corev1.Secret{} + if err := cl.Get(ctx, types.NamespacedName{ + Name: sel.Name, + Namespace: namespace, + }, secret); err != nil { + return nil, errors.Wrap(err, "get secret") + } + + value, ok := secret.Data[key] + if !ok { + return nil, errors.Errorf("key %s not found in secret %s", key, sel.Name) + } + + return value, nil +} + +func buildEncryptionKeySecretData(ctx context.Context, cl client.Client, cr *apiv1.PerconaServerMySQL) (map[string][]byte, error) { + data := make(map[string][]byte) + + if cr.Spec.Backup == nil { + return data, nil + } + + if cr.Spec.Backup.EncryptionKeySecret != nil { + key, err := readEncryptionKey(ctx, cl, *cr.Spec.Backup.EncryptionKeySecret, cr.Namespace) + if err != nil { + return nil, errors.Wrap(err, "read backup encryption key") + } + data[naming.InternalEncryptionKeyFileName(cr.GetName(), "")] = key + } + + for storageName, storage := range cr.Spec.Backup.Storages { + if storage == nil || storage.EncryptionKeySecret == nil { + continue + } + + key, err := readEncryptionKey(ctx, cl, *storage.EncryptionKeySecret, cr.Namespace) + if err != nil { + return nil, errors.Wrap(err, "read backup encryption key") + } + data[naming.InternalEncryptionKeyFileName(cr.GetName(), storageName)] = key + } + return data, nil +} + +// Reconcile the internal encryption key secret for the backup. +func (r *PerconaServerMySQLReconciler) reconcileInternalEncryptionKeySecret(ctx context.Context, cr *apiv1.PerconaServerMySQL) error { + if cr.Spec.Backup == nil || cr.CompareVersion("1.2.0") < 0 { + return nil + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: naming.EncryptionKeyInternalSecretName(cr.Name), + Namespace: cr.Namespace, + }, + } + + data, err := buildEncryptionKeySecretData(ctx, r.Client, cr) + if err != nil { + return errors.Wrap(err, "build encryption key secret data") + } + + if len(data) == 0 { + if err := r.Delete(ctx, secret); client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "delete encryption key secret") + } + return nil + } + + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + if err := controllerutil.SetControllerReference(cr, secret, r.Scheme); err != nil { + return errors.Wrap(err, "set controller reference") + } + secret.Labels = cr.GlobalLabels() + secret.Annotations = cr.GlobalAnnotations() + + if encryptionKeyDataEqual(secret.Data, data) { + return nil + } + + // Store a version of the data along with the keys. The version is the timestamp at which + // the data was last modified. This version is used to signal the sidecar whether to wait + // for the data to be synced from the Secret to the pod. + // The sidecar reads the version file from its mounted Secret volume; when it matches this value, + // kubelet has refreshed the projected data and the sidecar can safely use the + // encryption keys. + version := []byte(strconv.FormatInt(time.Now().UTC().UnixNano(), 10)) + + secret.Data = data + secret.Data[naming.InternalEncryptionKeyVersionFileName] = version + + return nil + }); err != nil { + return errors.Wrap(err, "create or update encryption key secret") + } + return nil +} + +func encryptionKeyDataEqual(current, desired map[string][]byte) bool { + if len(current) != len(desired)+1 { + return false + } + for key, desiredValue := range desired { + if !bytes.Equal(current[key], desiredValue) { + return false + } + } + _, ok := current[naming.InternalEncryptionKeyVersionFileName] + return ok +} diff --git a/pkg/controller/psbackup/controller.go b/pkg/controller/psbackup/controller.go index 6ef80f5e4..7c52d861f 100644 --- a/pkg/controller/psbackup/controller.go +++ b/pkg/controller/psbackup/controller.go @@ -349,10 +349,15 @@ func (r *PerconaServerMySQLBackupReconciler) prepareStatus( status.Destination = destination status.Image = cluster.Spec.Backup.Image - status.Storage = storage + status.Storage = storage.DeepCopy() status.BackupSource = backupSource status.Type = cr.Spec.Type status.Compressed = cr.IsCompressed(storage, cluster.Spec.MySQL.Configuration) + + if status.Storage.EncryptionKeySecret == nil { + status.Storage.EncryptionKeySecret = cluster.Spec.Backup.EncryptionKeySecret + } + return nil } @@ -448,6 +453,24 @@ func (r *PerconaServerMySQLBackupReconciler) createBackupJob( } } + if cluster.Spec.Backup.GetEncryptionEnabled(storage) { + secret := &corev1.Secret{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: cr.Namespace, + Name: naming.EncryptionKeyInternalSecretName(cluster.Name), + }, secret); err != nil { + return errors.Wrap(err, "get encryption key secret") + } + + version, ok := secret.Data[naming.InternalEncryptionKeyVersionFileName] + if !ok { + return errors.Errorf("version not found in encryption key secret %s/%s", secret.Namespace, secret.Name) + } + if err := xtrabackup.SetEncryptionKeyFileVersion(job, string(version)); err != nil { + return errors.Wrap(err, "set encryption key file version") + } + } + if err := controllerutil.SetControllerReference(cr, job, r.Scheme); err != nil { return errors.Wrapf(err, "set controller reference to Job %s/%s", job.Namespace, job.Name) } diff --git a/pkg/controller/psrestore/restorer.go b/pkg/controller/psrestore/restorer.go index 560a9caf7..26b45b1ce 100644 --- a/pkg/controller/psrestore/restorer.go +++ b/pkg/controller/psrestore/restorer.go @@ -382,6 +382,10 @@ func getBackup(ctx context.Context, cl client.Client, cr *apiv1.PerconaServerMyS } storage, ok := cluster.Spec.Backup.Storages[backup.Spec.StorageName] if ok { + storage = storage.DeepCopy() + if storage.EncryptionKeySecret == nil { + storage.EncryptionKeySecret = cluster.Spec.Backup.EncryptionKeySecret + } backup.Status.Storage = storage } return backup, nil diff --git a/pkg/mysql/mysql.go b/pkg/mysql/mysql.go index c5d024e4b..3fec8702b 100644 --- a/pkg/mysql/mysql.go +++ b/pkg/mysql/mysql.go @@ -319,7 +319,27 @@ func volumes(cr *apiv1.PerconaServerMySQL) []corev1.Volume { VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: cr.Spec.MySQL.VaultSecretName, - Optional: ptr.To(true), + Optional: new(true), + }, + }, + }) + } + + if cr.CompareVersion("1.2.0") >= 0 { + // This is the internal Secret managed by the operator. + // It is mounted unconditionally, even if empty, so that the mysql pods do not require a restart + // each time an encryption key is added/updated. + // The xtrabackup sidecar is long-running, so encrypted backups need the + // internal key Secret projected into the pod before a backup request arrives. + // The Secret is optional for clusters without backup encryption. When present, + // its version file lets the sidecar wait until kubelet has synced the mounted + // key data before starting xtrabackup. + volumes = append(volumes, corev1.Volume{ + Name: "backup-encryption-keys", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: naming.EncryptionKeyInternalSecretName(cr.Name), + Optional: new(true), }, }, }) @@ -768,6 +788,16 @@ func backupVolumeMounts(cr *apiv1.PerconaServerMySQL) []corev1.VolumeMount { ) } + if cr.CompareVersion("1.2.0") >= 0 { + // See the matching optional volume for how encryption keys propagate. + mounts = append(mounts, + corev1.VolumeMount{ + Name: "backup-encryption-keys", + MountPath: "/etc/mysql/encryption-keys", + }, + ) + } + return mounts } diff --git a/pkg/mysql/mysql_test.go b/pkg/mysql/mysql_test.go index 19bdb8c9b..524c96eea 100644 --- a/pkg/mysql/mysql_test.go +++ b/pkg/mysql/mysql_test.go @@ -518,6 +518,15 @@ func expectedVolumes() []corev1.Volume { }, }, }, + { + Name: "backup-encryption-keys", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "internal-encryption-keys-ps-cluster1", + Optional: ptr.To(true), + }, + }, + }, } } @@ -728,6 +737,7 @@ func TestBackupVolumeMounts(t *testing.T) { {Name: "backup-logs", MountPath: BackupLogDir}, {Name: vaultSecretVolumeName, MountPath: vaultSecretMountPath}, {Name: configVolumeName, MountPath: configMountPath}, + {Name: "backup-encryption-keys", MountPath: "/etc/mysql/encryption-keys"}, } mounts := backupVolumeMounts(cr) diff --git a/pkg/naming/backup.go b/pkg/naming/backup.go new file mode 100644 index 000000000..9de7c2ee0 --- /dev/null +++ b/pkg/naming/backup.go @@ -0,0 +1,15 @@ +package naming + +const InternalEncryptionKeyVersionFileName = ".version" + +func EncryptionKeyInternalSecretName(clusterName string) string { + return "internal-encryption-keys-" + clusterName +} + +func InternalEncryptionKeyFileName(clusterName, storageName string) string { + name := clusterName + if storageName != "" { + name = name + "-" + storageName + } + return name +} diff --git a/pkg/xtrabackup/xtrabackup.go b/pkg/xtrabackup/xtrabackup.go index 6276a3032..1997311aa 100644 --- a/pkg/xtrabackup/xtrabackup.go +++ b/pkg/xtrabackup/xtrabackup.go @@ -28,18 +28,20 @@ import ( ) const ( - appName = "xtrabackup" - componentShortName = "xb" - dataVolumeName = "datadir" - dataMountPath = "/var/lib/mysql" - credsVolumeName = "users" - credsMountPath = "/etc/mysql/mysql-users-secret" - tlsVolumeName = "tls" - tlsMountPath = "/etc/mysql/mysql-tls-secret" - backupVolumeName = appName - backupMountPath = "/backup" - vaultSecretVolumeName = "vault-keyring-secret" - vaultSecretMountPath = "/etc/mysql/vault-keyring-secret" + appName = "xtrabackup" + componentShortName = "xb" + dataVolumeName = "datadir" + dataMountPath = "/var/lib/mysql" + credsVolumeName = "users" + credsMountPath = "/etc/mysql/mysql-users-secret" + tlsVolumeName = "tls" + tlsMountPath = "/etc/mysql/mysql-tls-secret" + backupVolumeName = appName + backupMountPath = "/backup" + vaultSecretVolumeName = "vault-keyring-secret" + vaultSecretMountPath = "/etc/mysql/vault-keyring-secret" + encryptionKeysVolumeName = "backup-encryption-keys" + encryptionKeysMountPath = "/etc/mysql/encryption-keys" ) func Name(cr *apiv1.PerconaServerMySQLBackup) string { @@ -105,6 +107,25 @@ func trimJobName(name string) string { return name } +func encryptionKeyFileName(cluster *apiv1.PerconaServerMySQL, cr *apiv1.PerconaServerMySQLBackup) string { + for storageName, storage := range cluster.Spec.Backup.Storages { + if storageName != cr.Spec.StorageName { + continue + } + + if storage != nil && storage.EncryptionKeySecret != nil { + return naming.InternalEncryptionKeyFileName(cluster.GetName(), storageName) + } + + } + + if cluster.Spec.Backup.EncryptionKeySecret != nil { + return naming.InternalEncryptionKeyFileName(cluster.GetName(), "") + } + + return "" +} + func Job( cluster *apiv1.PerconaServerMySQL, cr *apiv1.PerconaServerMySQLBackup, @@ -222,7 +243,7 @@ func xtrabackupContainer(cluster *apiv1.PerconaServerMySQL, cr *apiv1.PerconaSer return corev1.Container{}, errors.Wrap(err, "marshal container options") } - return corev1.Container{ + container := corev1.Container{ Name: appName, Image: spec.Image, ImagePullPolicy: spec.ImagePullPolicy, @@ -263,7 +284,19 @@ func xtrabackupContainer(cluster *apiv1.PerconaServerMySQL, cr *apiv1.PerconaSer TerminationMessagePolicy: corev1.TerminationMessageReadFile, SecurityContext: storage.ContainerSecurityContext, Resources: storage.Resources, - }, nil + } + + if cluster.CompareVersion("1.2.0") >= 0 { + encryptionKeyFile := encryptionKeyFileName(cluster, cr) + if encryptionKeyFile != "" { + container.Env = append(container.Env, corev1.EnvVar{ + Name: "ENCRYPTION_KEY_FILE", + Value: path.Join(encryptionKeysMountPath, encryptionKeyFile), + }) + } + } + + return container, nil } type XBCloudAction string @@ -500,6 +533,23 @@ func RestoreJob( }) } + if storage.EncryptionKeySecret != nil { + job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: encryptionKeysVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: storage.EncryptionKeySecret.Name, + Items: []corev1.KeyToPath{ + { + Key: storage.EncryptionKeySecret.Key, + Path: "encryption-key", + }, + }, + }, + }, + }) + } + return job } @@ -600,6 +650,11 @@ func restoreContainer( ) } + encryptionAlgorithm := "AES256" + if e := restore.GetContainerOptions(storage).GetArgs().GetXtrabackupFlagValue("--encrypt"); e != "" { + encryptionAlgorithm = e + } + envs := util.MergeEnvLists( baseEnvs, restore.GetContainerOptions(storage).GetEnv(), @@ -626,6 +681,17 @@ func restoreContainer( }) } + if storage.EncryptionKeySecret != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: encryptionKeysVolumeName, + MountPath: encryptionKeysMountPath, + }) + envs = append(envs, corev1.EnvVar{ + Name: "ENCRYPTION_ALGORITHM", + Value: encryptionAlgorithm, + }) + } + return corev1.Container{ Name: appName, Image: spec.Image, @@ -856,14 +922,31 @@ func SetSourceNode(job *batchv1.Job, src string) error { return errors.Errorf("no container named %s in Job spec", appName) } +func SetEncryptionKeyFileVersion(job *batchv1.Job, ver string) error { + spec := &job.Spec.Template.Spec + + for i := range spec.Containers { + container := &spec.Containers[i] + + if container.Name == appName { + container.Env = append(container.Env, corev1.EnvVar{Name: "ENCRYPTION_KEY_FILE_VERSION", Value: ver}) + return nil + } + } + + return errors.Errorf("no container named %s in Job spec", appName) +} + type BackupConfig struct { - Destination string `json:"destination"` - Type apiv1.BackupStorageType `json:"type"` - VerifyTLS bool `json:"verifyTLS,omitempty"` - ContainerOptions *apiv1.BackupContainerOptions `json:"containerOptions,omitempty"` - S3 BackupConfigS3 `json:"s3"` - GCS BackupConfigGCS `json:"gcs"` - Azure BackupConfigAzure `json:"azure"` + Destination string `json:"destination"` + Type apiv1.BackupStorageType `json:"type"` + VerifyTLS bool `json:"verifyTLS,omitempty"` + ContainerOptions *apiv1.BackupContainerOptions `json:"containerOptions,omitempty"` + S3 BackupConfigS3 `json:"s3"` + GCS BackupConfigGCS `json:"gcs"` + Azure BackupConfigAzure `json:"azure"` + EncryptionKeyFile string `json:"encryptionKeyFile,omitempty"` + EncryptionKeyVersion string `json:"encryptionKeyVersion,omitempty"` // Specify for incremental backups IncrementalLsn string `json:"incrementalLsn,omitempty"`