K8SPS-409 | Encrypted backups

api/v1/perconaservermysql_types.go

@@ -316,6 +316,15 @@ type PMMSpec struct {
 	ReadinessProbe           *corev1.Probe               `json:"readinessProbe,omitempty"`
 }
 
+type EncryptionKeySecretSelector struct {
+	// +kubebuilder:validation:Required
+	Name string `json:"name"`
+
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:=encryptionKey
+	Key string `json:"key,omitempty"`
+}
+
 type BackupSpec struct {
 	Enabled                  bool                          `json:"enabled,omitempty"`
 	SourcePod                string                        `json:"sourcePod,omitempty"`
@@ -333,6 +342,19 @@ type BackupSpec struct {
 	// Deprecated: not supported since v0.12.0. Use initContainer instead
 	InitImage     string             `json:"initImage,omitempty"`
 	InitContainer *InitContainerSpec `json:"initContainer,omitempty"`
+
+	// EncryptionKeySecret is the secret key selector for the backup encryption key.
+	EncryptionKeySecret *EncryptionKeySecretSelector `json:"encryptionKeySecret,omitempty"`
+}
+
+func (s *BackupSpec) GetEncryptionEnabled(storage *BackupStorageSpec) bool {
+	if s.EncryptionKeySecret != nil {
+		return true
+	}
+	if storage != nil && storage.EncryptionKeySecret != nil {
+		return true
+	}
+	return false
 }
 
 type BackupSchedule struct {
@@ -388,6 +410,11 @@ type BackupStorageSpec struct {
 	RuntimeClassName          *string                           `json:"runtimeClassName,omitempty"`
 	VerifyTLS                 *bool                             `json:"verifyTLS,omitempty"`
 	ContainerOptions          *BackupContainerOptions           `json:"containerOptions,omitempty"`
+
+	// EncryptionKeySecret is the secret key selector for the backup encryption key.
+	// This takes precedence over the encryption key secret in the backup spec.
+	// +optional
+	EncryptionKeySecret *EncryptionKeySecretSelector `json:"encryptionKeySecret,omitempty"`
 }
 
 type BackupContainerOptions struct {
@@ -404,6 +431,25 @@ func (b *BackupContainerOptions) GetEnv() []corev1.EnvVar {
 	return util.MergeEnvLists(b.Env, b.Args.env())
 }
 
+func (b *BackupContainerOptions) GetArgs() BackupContainerArgs {
+	if b == nil {
+		return BackupContainerArgs{}
+	}
+	return b.Args
+}
+
+func (args BackupContainerArgs) GetXtrabackupFlagValue(flag string) string {
+	matchPrefix := flag + "="
+	for _, arg := range args.Xtrabackup {
+		if !strings.HasPrefix(arg, matchPrefix) {
+			continue
+		}
+
+		return strings.TrimPrefix(arg, matchPrefix)
+	}
+	return ""
+}
+
 type BackupContainerArgs struct {
 	Xtrabackup []string `json:"xtrabackup,omitempty"`
 	Xbcloud    []string `json:"xbcloud,omitempty"`

api/v1/perconaservermysql_types_test.go

@@ -752,3 +752,59 @@ func TestBackupStorageSpecEquals(t *testing.T) {
 		})
 	}
 }
+
+func TestGetXtrabackupFlagValue(t *testing.T) {
+	testCases := []struct {
+		containerOptions BackupContainerOptions
+		flag             string
+		want             string
+	}{
+		{
+			containerOptions: BackupContainerOptions{
+				Args: BackupContainerArgs{
+					Xtrabackup: []string{"--compress=lz4"},
+				},
+			},
+			flag: "--compress",
+			want: "lz4",
+		},
+		{
+			containerOptions: BackupContainerOptions{
+				Args: BackupContainerArgs{
+					Xtrabackup: []string{"--encrypt=AES256"},
+				},
+			},
+			flag: "--encrypt",
+			want: "AES256",
+		},
+		{
+			containerOptions: BackupContainerOptions{
+				Args: BackupContainerArgs{
+					Xtrabackup: []string{"--encrypt=AES256", "--encrypt-key-file=encryption-key"},
+				},
+			},
+			flag: "--encrypt",
+			want: "AES256",
+		},
+		{
+			containerOptions: BackupContainerOptions{
+				Args: BackupContainerArgs{
+					Xtrabackup: []string{"--encrypt=AES256", "--encrypt-key-file=encryption-key"},
+				},
+			},
+			flag: "--encrypt-key-file",
+			want: "encryption-key",
+		},
+		{
+			containerOptions: BackupContainerOptions{},
+			flag:             "--encrypt",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.flag, func(t *testing.T) {
+			got := tc.containerOptions.GetArgs().GetXtrabackupFlagValue(tc.flag)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}

build/run-backup.sh

@@ -14,6 +14,8 @@ request_data() {
 				    "containerOptions": ${CONTAINER_OPTIONS},
 				    "verifyTLS": $(json_escape "${VERIFY_TLS}"),
 					"incrementalLsn": "$(json_escape "${INCREMENTAL_LSN}")",
+					"encryptionKeyFile": "$(json_escape "${ENCRYPTION_KEY_FILE}")",
+					"encryptionKeyVersion": "$(json_escape "${ENCRYPTION_KEY_FILE_VERSION}")",
 				    "s3": {
 				        "bucket": "$(json_escape "${S3_BUCKET}")",
 				        "endpointUrl": "$(json_escape "${AWS_ENDPOINT}")",
@@ -32,6 +34,8 @@ request_data() {
 				    "type": "$(json_escape "${STORAGE_TYPE}")",
 				    "containerOptions": ${CONTAINER_OPTIONS},
 					"incrementalLsn": "$(json_escape "${INCREMENTAL_LSN}")",
+					"encryptionKeyFile": "$(json_escape "${ENCRYPTION_KEY_FILE}")",
+					"encryptionKeyVersion": "$(json_escape "${ENCRYPTION_KEY_FILE_VERSION}")",
 				    "gcs": {
 				        "bucket": "$(json_escape "${GCS_BUCKET}")",
 				        "endpointUrl": "$(json_escape "${GCS_ENDPOINT}")",
@@ -50,6 +54,8 @@ request_data() {
 				    "type": "$(json_escape "${STORAGE_TYPE}")",
 				    "containerOptions": ${CONTAINER_OPTIONS},
 					"incrementalLsn": "$(json_escape "${INCREMENTAL_LSN}")",
+					"encryptionKeyFile": "$(json_escape "${ENCRYPTION_KEY_FILE}")",
+					"encryptionKeyVersion": "$(json_escape "${ENCRYPTION_KEY_FILE_VERSION}")",
 				    "azure": {
 				        "containerName": "$(json_escape "${AZURE_CONTAINER_NAME}")",
 				        "storageAccount": "$(json_escape "${AZURE_STORAGE_ACCOUNT}")",

build/run-restore.sh

@@ -3,6 +3,8 @@
 set -e
 set -o xtrace
 
+encryption_key_file="/etc/mysql/encryption-keys/encryption-key"
+
 XTRABACKUP_VERSION=$(xtrabackup --version 2>&1 | awk '/^xtrabackup version/{print $3}' | awk -F'.' '{print $1"."$2}')
 DATADIR=${DATADIR:-/var/lib/mysql}
 PARALLEL=$(grep -c processor /proc/cpuinfo)
@@ -41,6 +43,15 @@ extract() {
 	xbstream -xv -C "${targetdir}" --parallel="${PARALLEL}" ${XBSTREAM_EXTRA_ARGS}
 }
 
+decrypt() {
+	local targetdir=$1
+	if [ -n "${ENCRYPTION_ALGORITHM}" ]; then
+		# shellcheck disable=SC2086
+		xtrabackup --decrypt=${ENCRYPTION_ALGORITHM} --encrypt-key-file=${encryption_key_file} --target-dir="${targetdir}" --parallel="${PARALLEL}" ${XB_EXTRA_ARGS}
+		find "${targetdir}" -name '*.xbcrypt' -delete
+	fi
+}
+
 get_keyring_arg() {
 	local keyring=""
 	if [[ -f ${KEYRING_VAULT_PATH} ]]; then
@@ -63,7 +74,9 @@ restore_full() {
 	rm -rf "${DATADIR:?}"/*
 	tmpdir=$(mktemp --directory "${DATADIR}/${RESTORE_NAME}_XXXX")
 
+	echo "Downloading backup: ${BACKUP_DEST}"
 	download "${BACKUP_DEST}" | extract "${tmpdir}"
+	decrypt "${tmpdir}"
 
 	# shellcheck disable=SC2086
 	xtrabackup --decompress --remove-original --parallel="${PARALLEL}" --target-dir="${tmpdir}" ${XB_EXTRA_ARGS}
@@ -93,6 +106,7 @@ restore_incremental() {
 	# Download and extract the base (full) backup
 	echo "Downloading base backup: ${BACKUP_DEST}"
 	download "${BACKUP_DEST}" | extract "${basedir}"
+	decrypt "${basedir}"
 
 	# shellcheck disable=SC2086
 	xtrabackup --decompress --remove-original --parallel="${PARALLEL}" --target-dir="${basedir}" ${XB_EXTRA_ARGS}
@@ -117,6 +131,7 @@ restore_incremental() {
 		incrdir=$(mktemp --directory "${DATADIR}/${RESTORE_NAME}_incr${count}_XXXX")
 
 		download "${incr_dest}" | extract "${incrdir}"
+		decrypt "${incrdir}"
 
 		# shellcheck disable=SC2086
 		xtrabackup --decompress --remove-original --parallel="${PARALLEL}" --target-dir="${incrdir}" ${XB_EXTRA_ARGS}

cmd/example-gen/main.go

@@ -213,6 +213,10 @@ func printRestore() error {
 					NodeSelector:      defaults.NodeSelector,
 					VerifyTLS:         defaults.VerifyTLS,
 					RuntimeClassName:  defaults.RuntimeClassName,
+					EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{
+						Name: "my-encryption-key-secret",
+						Key:  "encryptionKey",
+					},
 				},
 			},
 		},

cmd/example-gen/pkg/defaults/manual.go

@@ -147,6 +147,11 @@ func backupDefaults(spec *apiv1.BackupSpec) {
 			LogLevel:           "info",
 		},
 	}
+
+	spec.EncryptionKeySecret = &apiv1.EncryptionKeySecretSelector{
+		Name: "my-encryption-key-secret",
+		Key:  "encryptionKey",
+	}
 	podSpecDefaults(&spec.PiTR.BinlogServer.PodSpec, ImageBinlogServer, corev1.ResourceRequirements{}, "", 30, nil, nil)
 	spec.PiTR.BinlogServer.Size = 1
 	spec.SourcePod = SourcePod
@@ -176,6 +181,10 @@ func backupDefaults(spec *apiv1.BackupSpec) {
 				EndpointURL:       "https://accountName.blob.core.windows.net",
 				StorageClass:      "Cool",
 			},
+			EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{
+				Name: "my-azure-encryption-key-secret",
+				Key:  "encryptionKey",
+			},
 		},
 		"gcp-cs": {
 			Type: apiv1.BackupStorageGCS,
@@ -185,6 +194,10 @@ func backupDefaults(spec *apiv1.BackupSpec) {
 				CredentialsSecret: "SECRET-NAME",
 				EndpointURL:       "https://storage.googleapis.com",
 			},
+			EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{
+				Name: "my-gcs-encryption-key-secret",
+				Key:  "encryptionKey",
+			},
 		},
 		"s3-us-west": {
 			Type: apiv1.BackupStorageS3,
@@ -206,6 +219,10 @@ func backupDefaults(spec *apiv1.BackupSpec) {
 			SchedulerName:     SchedulerName,
 			VerifyTLS:         VerifyTLS,
 			NodeSelector:      NodeSelector,
+			EncryptionKeySecret: &apiv1.EncryptionKeySecretSelector{
+				Name: "my-s3-encryption-key-secret",
+				Key:  "encryptionKey",
+			},
 		},
 	}
 }

cmd/example-gen/scripts/lib/ps.sh

@@ -220,6 +220,7 @@ del_fields_to_comment() {
 		| yq "del(.spec.backup.containerSecurityContext)" \
 		| yq "del(.spec.backup.resources)" \
 		| yq "del(.spec.backup.serviceAccountName)" \
+		| yq "del(.spec.backup.encryptionKeySecret)" \
 		| yq "del(.spec.backup.storages.azure-blob)" \
 		| yq "del(.spec.backup.storages.gcp-cs)" \
 		| yq "del(.spec.backup.storages.s3-us-west.resources)" \
@@ -238,6 +239,7 @@ del_fields_to_comment() {
 		| yq "del(.spec.backup.storages.s3-us-west.s3.endpointUrl)" \
 		| yq "del(.spec.backup.storages.s3-us-west.schedulerName)" \
 		| yq "del(.spec.backup.storages.s3-us-west.runtimeClassName)" \
+		| yq "del(.spec.backup.storages.s3-us-west.encryptionKeySecret)" \
 		| yq "del(.spec.toolkit.imagePullSecrets)" \
 		| yq "del(.spec.toolkit.env)" \
 		| yq "del(.spec.toolkit.envFrom)" \