K8SPS-409 | Encrypted backups

api/v1/perconaservermysql_types.go

@@ -35,6 +35,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -359,6 +360,30 @@ type BackupStorageSpec struct {
 	RuntimeClassName          *string                           `json:"runtimeClassName,omitempty"`
 	VerifyTLS                 *bool                             `json:"verifyTLS,omitempty"`
 	ContainerOptions          *BackupContainerOptions           `json:"containerOptions,omitempty"`
+	Encryption                *EncryptionSpec                   `json:"encryption,omitempty"`
+}
+
+type EncryptionSpec struct {
+	Enabled    bool   `json:"enabled,omitempty"`
+	SecretName string `json:"secretName,omitempty"`
+	Key        string `json:"key,omitempty"`
+}
+
+func (e *EncryptionSpec) ReadEncryptionKey(ctx context.Context, cl client.Reader, namespace string) (string, error) {
+	if !e.Enabled {
+		return "", fmt.Errorf("encryption is not enabled")
+	}
+
+	secret := &corev1.Secret{}
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: e.SecretName}, secret); err != nil {
+		return "", errors.Wrap(err, "get encryption secret")
+	}
+
+	value := secret.Data[e.Key]
+	if len(value) == 0 {
+		return "", fmt.Errorf("encryption key not found in secret")
+	}
+	return string(value), nil
 }
 
 type BackupContainerOptions struct {

api/v1/perconaservermysqlbackup_types.go

@@ -31,6 +31,7 @@ type PerconaServerMySQLBackupSpec struct {
 	StorageName      string                  `json:"storageName"`
 	SourcePod        string                  `json:"sourcePod,omitempty"`
 	ContainerOptions *BackupContainerOptions `json:"containerOptions,omitempty"`
+	Encryption       *EncryptionSpec         `json:"encryption,omitempty"`
 }
 
 type BackupState string
@@ -155,6 +156,16 @@ func (b *PerconaServerMySQLBackup) GetContainerOptions(storage *BackupStorageSpe
 	return nil
 }
 
+func (b *PerconaServerMySQLBackup) GetEncryption(storage *BackupStorageSpec) *EncryptionSpec {
+	if b.Spec.Encryption != nil {
+		return b.Spec.Encryption
+	}
+	if storage != nil && storage.Encryption != nil {
+		return storage.Encryption
+	}
+	return nil
+}
+
 //+kubebuilder:object:root=true
 
 // PerconaServerMySQLBackupList contains a list of PerconaServerMySQLBackup

build/run-backup.sh

@@ -12,6 +12,7 @@
 				    "destination": "$(json_escape "${BACKUP_DEST}")",
 				    "type": "$(json_escape "${STORAGE_TYPE}")",
 				    "containerOptions": ${CONTAINER_OPTIONS},
+					"encryption": ${ENCRYPTION_OPTIONS},
 				    "verifyTLS": $(json_escape "${VERIFY_TLS}"),
 				    "s3": {
 				        "bucket": "$(json_escape "${S3_BUCKET}")",
@@ -30,6 +31,7 @@
 				    "verifyTLS": $(json_escape "${VERIFY_TLS}"),
 				    "type": "$(json_escape "${STORAGE_TYPE}")",
 				    "containerOptions": ${CONTAINER_OPTIONS},
+					"encryption": ${ENCRYPTION_OPTIONS},
 				    "gcs": {
 				        "bucket": "$(json_escape "${GCS_BUCKET}")",
 				        "endpointUrl": "$(json_escape "${GCS_ENDPOINT}")",
@@ -47,6 +49,7 @@
 				    "verifyTLS": $(json_escape "${VERIFY_TLS}"),
 				    "type": "$(json_escape "${STORAGE_TYPE}")",
 				    "containerOptions": ${CONTAINER_OPTIONS},
+					"encryption": ${ENCRYPTION_OPTIONS},
 				    "azure": {
 				        "containerName": "$(json_escape "${AZURE_CONTAINER_NAME}")",
 				        "storageAccount": "$(json_escape "${AZURE_STORAGE_ACCOUNT}")",
@@ -62,7 +65,7 @@
 
 # json_escape takes a string and replaces `\` to `\\` and `"` to `\"` to make it safe to insert provided argument into a json string
 json_escape() {
 	escaped_backslash=${1//'\'/'\\'}
 	escaped_quotes=${escaped_backslash//'"'/'\"'}
 	echo -n "$escaped_quotes"
 }

build/run-restore.sh

@@ -12,15 +12,15 @@
 fi
 
 run_s3() {
 	xbcloud get ${XBCLOUD_ARGS} "${BACKUP_DEST}" --storage=s3 --s3-bucket="${S3_BUCKET}"
 }
 
 run_gcs() {
 	xbcloud get ${XBCLOUD_ARGS} "${BACKUP_DEST}" --storage=google --google-bucket="${GCS_BUCKET}"
 }
 
 run_azure() {
 	xbcloud get ${XBCLOUD_ARGS} "${BACKUP_DEST}" --storage=azure
 }
 
 extract() {
@@ -43,6 +43,14 @@
 		"azure") run_azure | extract "${tmpdir}" ;;
 	esac
 
+	rm -rf "${tmpdir}/lost+found"
+
+	if [[ -n "${ENCRYPTION_KEY_FILE}" ]]; then
+		echo "Decrypting backup with key file: ${ENCRYPTION_KEY_FILE}"
+		xtrabackup --decrypt=AES256 --encrypt-key-file="${ENCRYPTION_KEY_FILE}" --target-dir="${tmpdir}" --parallel="${PARALLEL}"
+		find "${tmpdir}" -name '*.xbcrypt' -delete
+	fi
+
 	local keyring=""
 	if [[ -f ${KEYRING_VAULT_PATH} ]]; then
 		if [[ ${XTRABACKUP_VERSION} == "8.0" ]]; then
@@ -50,7 +58,7 @@
 			keyring="--keyring-vault-config=${KEYRING_VAULT_PATH}"
 		elif [[ ${XTRABACKUP_VERSION} == "8.4" ]]; then
 			# PXB expects the config with a specific name
 			cp ${KEYRING_VAULT_PATH} /tmp/component_keyring_vault.cnf
 			echo "Using keyring vault component: /tmp/component_keyring_vault.cnf"
 			keyring="--component-keyring-config=/tmp/component_keyring_vault.cnf"
 		fi

cmd/sidecar/handler/backup/create.go

@@ -92,9 +92,18 @@ func (h *Handler) createBackupHandler(w http.ResponseWriter, req *http.Request)
 		http.Error(w, "backup failed", http.StatusInternalServerError)
 		return
 	}
+
+	encryptionKeyFile, err := h.createEncryptionKeyFile(req.Context(), &backupConf, ns)
+	if err != nil {
+		log.Error(err, "failed to create encryption key file")
+		http.Error(w, "backup failed", http.StatusInternalServerError)
+		return
+	}
+	defer os.Remove(encryptionKeyFile) //nolint:errcheck
+
 	g, gCtx := errgroup.WithContext(req.Context())
 
-	xtrabackup := exec.CommandContext(gCtx, "xtrabackup", xtrabackupArgs(string(backupUser), backupPass, &backupConf)...)
+	xtrabackup := exec.CommandContext(gCtx, "xtrabackup", xtrabackupArgs(string(backupUser), backupPass, &backupConf, encryptionKeyFile)...)
 	xtrabackup.Env = envs(backupConf)
 
 	xbOut, err := xtrabackup.StdoutPipe()
@@ -199,7 +208,7 @@ func (h *Handler) createBackupHandler(w http.ResponseWriter, req *http.Request)
 	log.Info("Backup finished successfully", "destination", backupConf.Destination, "storage", backupConf.Type)
 }
 
-func xtrabackupArgs(user, pass string, conf *xb.BackupConfig) []string {
+func xtrabackupArgs(user, pass string, conf *xb.BackupConfig, encryptionKeyFile string) []string {
 	args := []string{
 		"--backup",
 		"--stream=xbstream",
@@ -212,6 +221,11 @@ func xtrabackupArgs(user, pass string, conf *xb.BackupConfig) []string {
 	if conf != nil && conf.ContainerOptions != nil {
 		args = append(args, conf.ContainerOptions.Args.Xtrabackup...)
 	}
+
+	if encryptionKeyFile != "" {
+		args = append(args, "--encrypt=AES256")
+		args = append(args, fmt.Sprintf("--encrypt-key-file=%s", encryptionKeyFile))
+	}
 	return args
 }
 
@@ -224,6 +238,31 @@ func getClusterType() apiv1.ClusterType {
 	return apiv1.ClusterType(cType)
 }
 
+func (h *Handler) createEncryptionKeyFile(
+	ctx context.Context, cfg *xb.BackupConfig, ns string) (string, error) {
+	if cfg.Encryption == nil || !cfg.Encryption.Enabled {
+		return "", nil
+	}
+
+	key, err := cfg.Encryption.ReadEncryptionKey(ctx, h.k8sClient, ns)
+	if err != nil {
+		return "", errors.Wrap(err, "read encryption key")
+	}
+
+	tempFile, err := os.CreateTemp(os.TempDir(), "encryption-key-*.key")
+	if err != nil {
+		return "", errors.Wrap(err, "create temporary file")
+	}
+
+	if _, err := tempFile.WriteString(key); err != nil {
+		return "", errors.Wrap(err, "write key to file")
+	}
+	if err := tempFile.Close(); err != nil {
+		return "", errors.Wrap(err, "close file")
+	}
+	return tempFile.Name(), nil
+}
+
 func (h *Handler) checkBackupMD5Size(ctx context.Context, cfg *xb.BackupConfig) error {
 	// xbcloud doesn't create md5 file for azure
 	if cfg.Type == apiv1.BackupStorageAzure {

cmd/sidecar/handler/backup/handler.go

@@ -8,6 +8,8 @@ import (
 	"sync/atomic"
 
 	"github.com/pkg/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
 	xb "github.com/percona/percona-server-mysql-operator/pkg/xtrabackup"
 	"github.com/percona/percona-server-mysql-operator/pkg/xtrabackup/storage"
@@ -19,9 +21,15 @@ type Handler struct {
 	newStorageFunc   storage.NewClientFunc
 	getNamespaceFunc func() (string, error)
 	deleteBackupFunc func(ctx context.Context, cfg *xb.BackupConfig, backupName string) error
+	k8sClient        client.Client
 }
 
-func (h *Handler) init() {
+func NewHandler() (*Handler, error) {
+	h := &Handler{}
+	return h, h.init()
+}
+
+func (h *Handler) init() error {
 	if h.deleteBackupFunc == nil {
 		h.deleteBackupFunc = deleteBackup
 	}
@@ -38,11 +46,19 @@ func (h *Handler) init() {
 			return string(ns), nil
 		}
 	}
+
+	if h.k8sClient == nil {
+		k8sClient, err := client.New(config.GetConfigOrDie(), client.Options{})
+		if err != nil {
+			return errors.Wrap(err, "new k8s client")
+		}
+		h.k8sClient = k8sClient
+	}
+
+	return nil
 }
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	h.init()
-
 	switch req.Method {
 	case http.MethodGet:
 		h.getBackupHandler(w, req)

cmd/sidecar/handler/handler.go

@@ -8,14 +8,9 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/percona/percona-server-mysql-operator/cmd/sidecar/handler/backup"
 	"github.com/percona/percona-server-mysql-operator/pkg/mysql"
 )
 
-func Backup() http.Handler {
-	return new(backup.Handler)
-}
-
 func LogsHandlerFunc(w http.ResponseWriter, req *http.Request) {
 	path := strings.Split(req.URL.Path, "/")
 	if len(path) < 3 {

cmd/sidecar/main.go

@@ -14,17 +14,24 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"github.com/percona/percona-server-mysql-operator/cmd/sidecar/handler"
+	"github.com/percona/percona-server-mysql-operator/cmd/sidecar/handler/backup"
 	"github.com/percona/percona-server-mysql-operator/pkg/mysql"
 )
 
 func startServer() *http.Server {
 	log := logf.Log.WithName("startServer")
 	mux := http.NewServeMux()
 
+	backupHandler, err := backup.NewHandler()
+	if err != nil {
+		log.Error(err, "failed to create backup handler")
+		os.Exit(1)
+	}
+
 	mux.HandleFunc("/health/", func(w http.ResponseWriter, req *http.Request) {
 		fmt.Fprintf(w, "OK")
 	})
-	mux.Handle("/backup/", handler.Backup())
+	mux.Handle("/backup/", backupHandler)
 	mux.HandleFunc("/logs/", handler.LogsHandlerFunc)
 
 	srv := &http.Server{Addr: ":" + strconv.Itoa(mysql.SidecarHTTPPort), Handler: mux}

config/crd/bases/ps.percona.com_perconaservermysqlbackups.yaml

@@ -148,6 +148,15 @@ spec:
                       type: object
                     type: array
                 type: object
+              encryption:
+                properties:
+                  enabled:
+                    type: boolean
+                  key:
+                    type: string
+                  secretName:
+                    type: string
+                type: object
               sourcePod:
                 type: string
               storageName:
@@ -805,6 +814,15 @@ spec:
                             type: string
                         type: object
                     type: object
+                  encryption:
+                    properties:
+                      enabled:
+                        type: boolean
+                      key:
+                        type: string
+                      secretName:
+                        type: string
+                    type: object
                   gcs:
                     properties:
                       bucket:

config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml

@@ -686,6 +686,15 @@ spec:
                                 type: string
                             type: object
                         type: object
+                      encryption:
+                        properties:
+                          enabled:
+                            type: boolean
+                          key:
+                            type: string
+                          secretName:
+                            type: string
+                        type: object
                       gcs:
                         properties:
                           bucket:

config/crd/bases/ps.percona.com_perconaservermysqls.yaml

@@ -2212,6 +2212,15 @@ spec:
                                   type: string
                               type: object
                           type: object
+                        encryption:
+                          properties:
+                            enabled:
+                              type: boolean
+                            key:
+                              type: string
+                            secretName:
+                              type: string
+                          type: object
                         gcs:
                           properties:
                             bucket:

deploy/backup/backup.yaml

@@ -19,3 +19,4 @@ spec:
 #        - --someflag=abc
 #      xtrabackup:
 #        - --someflag=abc
+  encryption: null

deploy/backup/restore.yaml

@@ -47,6 +47,7 @@ spec:
 #        privileged: false
 #        runAsGroup: 1001
 #        runAsUser: 1001
+#      encryption: null
 #      labels:
 #        rack: rack-22
 #      nodeSelector: