From 8c2aa9d81e215afdc30df2b5a397a9fda80af8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 12:38:50 +0300 Subject: [PATCH 001/102] fix binlog server deployment --- pkg/binlogserver/binlog_server.go | 21 +++++++++--- pkg/binlogserver/config.go | 57 +++++++++++++++++++++++++------ pkg/controller/ps/controller.go | 30 +++++++++++++--- 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/pkg/binlogserver/binlog_server.go b/pkg/binlogserver/binlog_server.go index a45ba5b04..7d85ee5a4 100644 --- a/pkg/binlogserver/binlog_server.go +++ b/pkg/binlogserver/binlog_server.go @@ -6,6 +6,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" "github.com/percona/percona-server-mysql-operator/pkg/k8s" @@ -18,7 +19,9 @@ const ( credsVolumeName = "users" CredsMountPath = "/etc/mysql/mysql-users-secret" tlsVolumeName = "tls" - tlsMountPath = "/etc/mysql/mysql-tls-secret" + TLSMountPath = "/etc/mysql/mysql-tls-secret" + bufferVolumeName = "buffer" + BufferMountPath = "/var/lib/binlogsrv" configVolumeName = "config" configMountPath = "/etc/binlog_server/config" storageCredsVolumeName = "storage" @@ -67,7 +70,7 @@ func StatefulSet(cr *apiv1.PerconaServerMySQL, initImage, configHash string) *ap Annotations: cr.GlobalAnnotations(), }, Spec: appsv1.StatefulSetSpec{ - Replicas: &spec.Size, + Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -112,6 +115,12 @@ func volumes(cr *apiv1.PerconaServerMySQL) []corev1.Volume { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: bufferVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, { Name: credsVolumeName, VolumeSource: corev1.VolumeSource{ @@ -212,15 +221,19 @@ func binlogServerContainer(cr *apiv1.PerconaServerMySQL) corev1.Container { }, { Name: tlsVolumeName, - MountPath: tlsMountPath, + MountPath: TLSMountPath, }, { Name: configVolumeName, MountPath: configMountPath, }, + { + Name: bufferVolumeName, + MountPath: BufferMountPath, + }, }, Command: []string{"/opt/percona/binlog-server-entrypoint.sh"}, - Args: []string{"/usr/local/bin/binlog_server", "pull", path.Join(configMountPath, ConfigKey)}, + Args: []string{"/usr/bin/binlog_server", "pull", path.Join(configMountPath, ConfigKey)}, TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: corev1.TerminationMessageReadFile, SecurityContext: spec.ContainerSecurityContext, diff --git a/pkg/binlogserver/config.go b/pkg/binlogserver/config.go index 0042bd532..a95218576 100644 --- a/pkg/binlogserver/config.go +++ b/pkg/binlogserver/config.go @@ -12,23 +12,60 @@ type Logger struct { File string `json:"file,omitempty"` } +type ConnectionSSL struct { + Mode string `json:"mode,omitempty"` + CA string `json:"ca,omitempty"` + CAPath string `json:"capath,omitempty"` + CRL string `json:"crl,omitempty"` + CRLPath string `json:"crlpath,omitempty"` + Cert string `json:"cert,omitempty"` + Key string `json:"key,omitempty"` + Cipher string `json:"cipher,omitempty"` +} + +type ConnectionTLS struct { + CipherSuites string `json:"ciphersuites,omitempty"` + Version string `json:"version,omitempty"` +} + type Connection struct { - Host string `json:"host,omitempty"` - Port int32 `json:"port,omitempty"` - User string `json:"user,omitempty"` - Password string `json:"password,omitempty"` - ConnectTimeout int32 `json:"connect_timeout,omitempty"` - ReadTimeout int32 `json:"read_timeout,omitempty"` - WriteTimeout int32 `json:"write_timeout,omitempty"` + Host string `json:"host,omitempty"` + Port int32 `json:"port,omitempty"` + User string `json:"user,omitempty"` + Password string `json:"password,omitempty"` + ConnectTimeout int32 `json:"connect_timeout,omitempty"` + ReadTimeout int32 `json:"read_timeout,omitempty"` + WriteTimeout int32 `json:"write_timeout,omitempty"` + SSL *ConnectionSSL `json:"ssl,omitempty"` + TLS *ConnectionTLS `json:"tls,omitempty"` +} + +type ReplicationMode string + +const ( + ReplicationModeGTID ReplicationMode = "gtid" + ReplicationModePosition ReplicationMode = "position" +) + +type Rewrite struct { + BaseFileName string `json:"base_file_name,omitempty"` + FileSize string `json:"file_size,omitempty"` } type Replication struct { - ServerID int32 `json:"server_id,omitempty"` - IdleTime int32 `json:"idle_time,omitempty"` + ServerID int32 `json:"server_id,omitempty"` + IdleTime int32 `json:"idle_time,omitempty"` + VerifyChecksum bool `json:"verify_checksum,omitempty"` + Mode ReplicationMode `json:"mode,omitempty"` + Rewrite Rewrite `json:"rewrite,omitempty"` } type Storage struct { - URI string `json:"uri,omitempty"` + Backend string `json:"backend,omitempty"` + URI string `json:"uri,omitempty"` + FsBufferDirectory string `json:"fs_buffer_directory,omitempty"` + CheckpointSize string `json:"checkpoint_size,omitempty"` + CheckpointInterval string `json:"checkpoint_interval,omitempty"` } type Configuration struct { diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index 62b67e427..7d330a3bf 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -21,6 +21,7 @@ import ( "crypto/md5" "encoding/json" "fmt" + "path" "slices" "strconv" "strings" @@ -1289,6 +1290,11 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context logger := logf.FromContext(ctx) + if cr.Status.MySQL.Ready < 1 { + logger.V(1).Info("Waiting for at least one MySQL pod to be ready") + return nil + } + if len(cr.Status.Host) == 0 { logger.V(1).Info("Waiting for .status.host to be populated") return nil @@ -1314,7 +1320,7 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context accessKey := s3Secret.Data[secret.CredentialsAWSAccessKey] secretKey := s3Secret.Data[secret.CredentialsAWSSecretKey] - s3Uri := fmt.Sprintf("s3://%s:%s@%s.%s", accessKey, secretKey, s3.Bucket, s3.Region) + s3Uri := fmt.Sprintf("https://%s:%s@%s", accessKey, secretKey, s3.EndpointURL) if len(s3.Prefix) > 0 { s3Uri += fmt.Sprintf("/%s", s3.Prefix) } @@ -1347,13 +1353,29 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context ConnectTimeout: cr.Spec.Backup.PiTR.BinlogServer.ConnectTimeout, WriteTimeout: cr.Spec.Backup.PiTR.BinlogServer.WriteTimeout, ReadTimeout: cr.Spec.Backup.PiTR.BinlogServer.ReadTimeout, + SSL: &binlogserver.ConnectionSSL{ + Mode: "verify_identity", + CA: path.Join(binlogserver.TLSMountPath, "ca.crt"), + Cert: path.Join(binlogserver.TLSMountPath, "tls.crt"), + Key: path.Join(binlogserver.TLSMountPath, "tls.key"), + }, }, Replication: binlogserver.Replication{ - ServerID: cr.Spec.Backup.PiTR.BinlogServer.ServerID, - IdleTime: cr.Spec.Backup.PiTR.BinlogServer.IdleTime, + Mode: binlogserver.ReplicationModeGTID, + ServerID: cr.Spec.Backup.PiTR.BinlogServer.ServerID, + IdleTime: cr.Spec.Backup.PiTR.BinlogServer.IdleTime, + VerifyChecksum: true, + Rewrite: binlogserver.Rewrite{ + BaseFileName: "binlog", + FileSize: "128M", + }, }, Storage: binlogserver.Storage{ - URI: s3Uri, + Backend: "s3", + URI: s3Uri, + CheckpointSize: "2M", + CheckpointInterval: "30s", + FsBufferDirectory: binlogserver.BufferMountPath, }, } From 1496ff36defb9339eaebe7545ad2a416812e61e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 12:44:21 +0300 Subject: [PATCH 002/102] remove embedded fields --- pkg/controller/psrestore/controller.go | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index c261ae974..3528df980 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -74,7 +74,7 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req log := logf.FromContext(ctx).WithName("PerconaServerMySQLRestore").WithValues("name", req.Name, "namespace", req.Namespace) cr := &apiv1.PerconaServerMySQLRestore{} - err := r.Client.Get(ctx, req.NamespacedName, cr) + err := r.Get(ctx, req.NamespacedName, cr) if err != nil { return ctrl.Result{}, errors.Wrapf(err, "get CR %s", req.NamespacedName) } @@ -95,18 +95,18 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req } err := k8sretry.OnError(k8sretry.DefaultRetry, retriable, func() error { cr := &apiv1.PerconaServerMySQLRestore{} - if err := r.Client.Get(ctx, req.NamespacedName, cr); err != nil { + if err := r.Get(ctx, req.NamespacedName, cr); err != nil { return errors.Wrapf(err, "get %v", req.NamespacedName.String()) } cr.Status = status log.Info("Updating status", "state", cr.Status.State) - if err := r.Client.Status().Update(ctx, cr); err != nil { + if err := r.Status().Update(ctx, cr); err != nil { return errors.Wrap(err, "update status") } - if err := r.Client.Get(ctx, req.NamespacedName, cr); err != nil { - return errors.Wrapf(err, "get %v", req.NamespacedName.String()) + if err := r.Get(ctx, req.NamespacedName, cr); err != nil { + return errors.Wrapf(err, "get %v", req.String()) } if cr.Status.State != status.State { return errors.Errorf("status %s was not updated to %s", cr.Status.State, status.State) @@ -128,7 +128,7 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req cluster := &apiv1.PerconaServerMySQL{} nn := types.NamespacedName{Name: cr.Spec.ClusterName, Namespace: cr.Namespace} - if err := r.Client.Get(ctx, nn, cluster); err != nil { + if err := r.Get(ctx, nn, cluster); err != nil { if k8serrors.IsNotFound(err) { status.State = apiv1.RestoreError status.StateDesc = fmt.Sprintf("PerconaServerMySQL %s in namespace %s is not found", cr.Spec.ClusterName, cr.Namespace) @@ -190,7 +190,7 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req job := &batchv1.Job{} nn = types.NamespacedName{Name: xtrabackup.RestoreJobName(cluster, cr), Namespace: req.Namespace} - err = r.Client.Get(ctx, nn, job) + err = r.Get(ctx, nn, job) if client.IgnoreNotFound(err) != nil { return ctrl.Result{}, errors.Wrapf(err, "get job %s", nn) } @@ -269,7 +269,7 @@ func (r *PerconaServerMySQLRestoreReconciler) deletePVCs(ctx context.Context, cl continue } - if err := r.Client.Delete(ctx, &pvc); err != nil { + if err := r.Delete(ctx, &pvc); err != nil { if !k8serrors.IsNotFound(err) { log.Error(err, "failed to delete PVC") } @@ -288,7 +288,7 @@ func (r *PerconaServerMySQLRestoreReconciler) removeBootstrapCondition(ctx conte err := k8sretry.RetryOnConflict(k8sretry.DefaultRetry, func() error { c := &apiv1.PerconaServerMySQL{} nn := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} - if err := r.Client.Get(ctx, nn, c); err != nil { + if err := r.Get(ctx, nn, c); err != nil { return err } @@ -311,13 +311,13 @@ func (r *PerconaServerMySQLRestoreReconciler) pauseCluster(ctx context.Context, err := k8sretry.RetryOnConflict(k8sretry.DefaultRetry, func() error { c := &apiv1.PerconaServerMySQL{} nn := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} - if err := r.Client.Get(ctx, nn, c); err != nil { + if err := r.Get(ctx, nn, c); err != nil { return err } c.Spec.Pause = true - if err := r.Client.Patch(ctx, c, client.MergeFrom(cluster)); err != nil { + if err := r.Patch(ctx, c, client.MergeFrom(cluster)); err != nil { return err } @@ -329,7 +329,7 @@ func (r *PerconaServerMySQLRestoreReconciler) pauseCluster(ctx context.Context, sts := &appsv1.StatefulSet{} nn := types.NamespacedName{Name: mysql.Name(cluster), Namespace: cluster.Namespace} - if err := r.Client.Get(ctx, nn, sts); err != nil { + if err := r.Get(ctx, nn, sts); err != nil { return errors.Wrapf(err, "get statefulset %s", nn) } @@ -340,7 +340,7 @@ func (r *PerconaServerMySQLRestoreReconciler) pauseCluster(ctx context.Context, switch cluster.Spec.MySQL.ClusterType { case apiv1.ClusterTypeAsync: nn = types.NamespacedName{Name: orchestrator.Name(cluster), Namespace: cluster.Namespace} - err := r.Client.Get(ctx, nn, sts) + err := r.Get(ctx, nn, sts) if client.IgnoreNotFound(err) != nil { return errors.Wrapf(err, "get statefulset %s", nn) } @@ -351,7 +351,7 @@ func (r *PerconaServerMySQLRestoreReconciler) pauseCluster(ctx context.Context, if cluster.HAProxyEnabled() { sts := new(appsv1.StatefulSet) nn = types.NamespacedName{Name: haproxy.Name(cluster), Namespace: cluster.Namespace} - if err := r.Client.Get(ctx, nn, sts); err != nil { + if err := r.Get(ctx, nn, sts); err != nil { return errors.Wrapf(err, "get deployment %s", nn) } if sts.Status.Replicas != 0 { @@ -362,7 +362,7 @@ func (r *PerconaServerMySQLRestoreReconciler) pauseCluster(ctx context.Context, if cluster.RouterEnabled() { deployment := new(appsv1.Deployment) nn = types.NamespacedName{Name: router.Name(cluster), Namespace: cluster.Namespace} - if err := r.Client.Get(ctx, nn, deployment); err != nil { + if err := r.Get(ctx, nn, deployment); err != nil { return errors.Wrapf(err, "get deployment %s", nn) } if deployment.Status.Replicas != 0 { From 925659c44f120310d6f4cf4f43f9e228a87be489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 12:51:58 +0300 Subject: [PATCH 003/102] add exec search functions --- pkg/binlogserver/binlog_server.go | 2 +- pkg/binlogserver/search.go | 83 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 pkg/binlogserver/search.go diff --git a/pkg/binlogserver/binlog_server.go b/pkg/binlogserver/binlog_server.go index 7d85ee5a4..df282dfcf 100644 --- a/pkg/binlogserver/binlog_server.go +++ b/pkg/binlogserver/binlog_server.go @@ -233,7 +233,7 @@ func binlogServerContainer(cr *apiv1.PerconaServerMySQL) corev1.Container { }, }, Command: []string{"/opt/percona/binlog-server-entrypoint.sh"}, - Args: []string{"/usr/bin/binlog_server", "pull", path.Join(configMountPath, ConfigKey)}, + Args: []string{binlogServerBinary, "pull", path.Join(configMountPath, ConfigKey)}, TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: corev1.TerminationMessageReadFile, SecurityContext: spec.ContainerSecurityContext, diff --git a/pkg/binlogserver/search.go b/pkg/binlogserver/search.go new file mode 100644 index 000000000..5c70adbb1 --- /dev/null +++ b/pkg/binlogserver/search.go @@ -0,0 +1,83 @@ +package binlogserver + +import ( + "bytes" + "context" + "encoding/json" + "path" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/clientcmd" + "github.com/percona/percona-server-mysql-operator/pkg/k8s" +) + +const binlogServerBinary = "/usr/bin/binlog_server" + +type SearchResponse struct { + Version int `json:"version"` + Status string `json:"status"` + Result []BinlogEntry `json:"result"` +} + +type BinlogEntry struct { + Name string `json:"name"` + Size int64 `json:"size"` + URI string `json:"uri"` + PreviousGTIDs string `json:"previous_gtids"` + AddedGTIDs string `json:"added_gtids"` + MinTimestamp string `json:"min_timestamp"` + MaxTimestamp string `json:"max_timestamp"` +} + +func SearchByGTID(ctx context.Context, cl client.Client, cliCmd clientcmd.Client, cr *apiv1.PerconaServerMySQL, gtidSet string) (*SearchResponse, error) { + return execSearch(ctx, cl, cliCmd, cr, "search_by_gtid_set", gtidSet) +} + +func SearchByTimestamp(ctx context.Context, cl client.Client, cliCmd clientcmd.Client, cr *apiv1.PerconaServerMySQL, timestamp string) (*SearchResponse, error) { + return execSearch(ctx, cl, cliCmd, cr, "search_by_timestamp", timestamp) +} + +func execSearch(ctx context.Context, cl client.Client, cliCmd clientcmd.Client, cr *apiv1.PerconaServerMySQL, subcommand, arg string) (*SearchResponse, error) { + pod, err := getBinlogServerPod(ctx, cl, cr) + if err != nil { + return nil, errors.Wrap(err, "get binlog server pod") + } + + configPath := path.Join(configMountPath, ConfigKey) + cmd := []string{binlogServerBinary, subcommand, configPath, arg} + + var stdout, stderr bytes.Buffer + if err := cliCmd.Exec(ctx, pod, AppName, cmd, nil, &stdout, &stderr, false); err != nil { + return nil, errors.Wrapf(err, "exec binlog_server %s: stderr: %s", subcommand, stderr.String()) + } + + var resp SearchResponse + if err := json.Unmarshal(stdout.Bytes(), &resp); err != nil { + return nil, errors.Wrapf(err, "unmarshal response: %s", stdout.String()) + } + + return &resp, nil +} + +func getBinlogServerPod(ctx context.Context, cl client.Client, cr *apiv1.PerconaServerMySQL) (*corev1.Pod, error) { + nn := types.NamespacedName{ + Namespace: cr.Namespace, + Name: Name(cr) + "-0", + } + + pod := &corev1.Pod{} + if err := cl.Get(ctx, nn, pod); err != nil { + return nil, errors.Wrapf(err, "get pod %s", nn) + } + + if !k8s.IsPodReady(*pod) { + return nil, errors.Errorf("binlog server pod %s is not ready", nn) + } + + return pod, nil +} From 634a30da3ec9cece6b58eea14d8864ba967e769c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 13:09:47 +0300 Subject: [PATCH 004/102] add pitr pkg --- api/v1/perconaservermysqlrestore_types.go | 14 ++ pkg/pitr/pitr.go | 240 ++++++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 pkg/pitr/pitr.go diff --git a/api/v1/perconaservermysqlrestore_types.go b/api/v1/perconaservermysqlrestore_types.go index 0c384755c..de7c824e1 100644 --- a/api/v1/perconaservermysqlrestore_types.go +++ b/api/v1/perconaservermysqlrestore_types.go @@ -31,8 +31,22 @@ type PerconaServerMySQLRestoreSpec struct { BackupName string `json:"backupName,omitempty"` BackupSource *PerconaServerMySQLBackupStatus `json:"backupSource,omitempty"` ContainerOptions *BackupContainerOptions `json:"containerOptions,omitempty"` + PITR *RestorePITRSpec `json:"pitr,omitempty"` } +type RestorePITRSpec struct { + Type PITRType `json:"type"` + Date string `json:"date,omitempty"` + GTID string `json:"gtid,omitempty"` +} + +type PITRType string + +const ( + PITRGtid PITRType = "gtid" + PITRDate PITRType = "date" +) + type RestoreState string const ( diff --git a/pkg/pitr/pitr.go b/pkg/pitr/pitr.go new file mode 100644 index 000000000..d7b5507a6 --- /dev/null +++ b/pkg/pitr/pitr.go @@ -0,0 +1,240 @@ +package pitr + +import ( + "fmt" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/k8s" + "github.com/percona/percona-server-mysql-operator/pkg/mysql" + "github.com/percona/percona-server-mysql-operator/pkg/naming" + "github.com/percona/percona-server-mysql-operator/pkg/secret" + "github.com/percona/percona-server-mysql-operator/pkg/util" +) + +const ( + appName = "pitr" + dataVolumeName = "datadir" + dataMountPath = "/var/lib/mysql" + credsVolumeName = "users" + credsMountPath = "/etc/mysql/mysql-users-secret" + tlsVolumeName = "tls" + tlsMountPath = "/etc/mysql/mysql-tls-secret" +) + +func JobName(cluster *apiv1.PerconaServerMySQL, restore *apiv1.PerconaServerMySQLRestore) string { + return fmt.Sprintf("pitr-restore-%s", restore.Name) +} + +func RestoreJob( + cluster *apiv1.PerconaServerMySQL, + restore *apiv1.PerconaServerMySQLRestore, + storage *apiv1.BackupStorageSpec, + initImage string, +) *batchv1.Job { + labels := util.SSMapMerge(cluster.GlobalLabels(), storage.Labels, restore.Labels(appName, naming.ComponentPITR)) + + pvcName := fmt.Sprintf("%s-%s-mysql-0", mysql.DataVolumeName, cluster.Name) + + return &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "batch/v1", + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: JobName(cluster, restore), + Namespace: cluster.Namespace, + Labels: labels, + Annotations: util.SSMapMerge(cluster.GlobalAnnotations(), storage.Annotations), + }, + Spec: batchv1.JobSpec{ + Parallelism: ptr.To(int32(1)), + Completions: ptr.To(int32(1)), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: cluster.GlobalAnnotations(), + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + ImagePullSecrets: cluster.Spec.Backup.ImagePullSecrets, + InitContainers: []corev1.Container{ + k8s.InitContainer( + cluster, + appName, + initImage, + cluster.Spec.Backup.InitContainer, + cluster.Spec.Backup.ImagePullPolicy, + storage.ContainerSecurityContext, + cluster.Spec.Backup.Resources, + []corev1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: dataMountPath, + }, + { + Name: credsVolumeName, + MountPath: credsMountPath, + }, + { + Name: tlsVolumeName, + MountPath: tlsMountPath, + }, + }, + ), + }, + Containers: []corev1.Container{ + restoreContainer(cluster, restore, storage), + }, + Affinity: storage.Affinity, + TopologySpreadConstraints: storage.TopologySpreadConstraints, + Tolerations: storage.Tolerations, + NodeSelector: storage.NodeSelector, + SchedulerName: storage.SchedulerName, + PriorityClassName: storage.PriorityClassName, + RuntimeClassName: storage.RuntimeClassName, + DNSPolicy: corev1.DNSClusterFirst, + SecurityContext: storage.PodSecurityContext, + Volumes: []corev1.Volume{ + { + Name: apiv1.BinVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: dataVolumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, + }, + { + Name: credsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.SecretsName, + }, + }, + }, + { + Name: tlsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.SSLSecretName, + }, + }, + }, + }, + }, + }, + BackoffLimit: cluster.Spec.Backup.BackoffLimit, + }, + } +} + +func restoreContainer( + cluster *apiv1.PerconaServerMySQL, + restore *apiv1.PerconaServerMySQLRestore, + storage *apiv1.BackupStorageSpec, +) corev1.Container { + binlogServer := cluster.Spec.Backup.PiTR.BinlogServer + + envs := []corev1.EnvVar{ + { + Name: "RESTORE_NAME", + Value: restore.Name, + }, + } + + if restore.Spec.PITR != nil { + envs = append(envs, corev1.EnvVar{ + Name: "PITR_TYPE", + Value: string(restore.Spec.PITR.Type), + }) + switch restore.Spec.PITR.Type { + case apiv1.PITRDate: + envs = append(envs, corev1.EnvVar{ + Name: "PITR_DATE", + Value: restore.Spec.PITR.Date, + }) + case apiv1.PITRGtid: + envs = append(envs, corev1.EnvVar{ + Name: "PITR_GTID", + Value: restore.Spec.PITR.GTID, + }) + } + } + + if binlogServer.Storage.S3 != nil { + s3 := binlogServer.Storage.S3 + bucket, _ := s3.BucketAndPrefix() + envs = append(envs, + corev1.EnvVar{ + Name: "STORAGE_TYPE", + Value: "s3", + }, + corev1.EnvVar{ + Name: "AWS_ACCESS_KEY_ID", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: k8s.SecretKeySelector(s3.CredentialsSecret, secret.CredentialsAWSAccessKey), + }, + }, + corev1.EnvVar{ + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: k8s.SecretKeySelector(s3.CredentialsSecret, secret.CredentialsAWSSecretKey), + }, + }, + corev1.EnvVar{ + Name: "AWS_DEFAULT_REGION", + Value: s3.Region, + }, + corev1.EnvVar{ + Name: "AWS_ENDPOINT", + Value: s3.EndpointURL, + }, + corev1.EnvVar{ + Name: "S3_BUCKET", + Value: bucket, + }, + ) + } + + envs = append(envs, restore.GetContainerOptions(storage).GetEnv()...) + + return corev1.Container{ + Name: appName, + Image: binlogServer.Image, + ImagePullPolicy: binlogServer.ImagePullPolicy, + Env: envs, + VolumeMounts: []corev1.VolumeMount{ + { + Name: apiv1.BinVolumeName, + MountPath: apiv1.BinVolumePath, + }, + { + Name: dataVolumeName, + MountPath: dataMountPath, + }, + { + Name: credsVolumeName, + MountPath: credsMountPath, + }, + { + Name: tlsVolumeName, + MountPath: tlsMountPath, + }, + }, + Command: []string{"/opt/percona/run-pitr-restore.sh"}, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + SecurityContext: storage.ContainerSecurityContext, + Resources: storage.Resources, + } +} From 6e76ed0954fb07c7b86909091cde444fbc007696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 13:31:31 +0300 Subject: [PATCH 005/102] run pitr job --- pkg/controller/psrestore/controller.go | 77 ++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index 3528df980..f475b0bcb 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -34,6 +34,7 @@ import ( k8sretry "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" @@ -41,6 +42,7 @@ import ( "github.com/percona/percona-server-mysql-operator/pkg/k8s" "github.com/percona/percona-server-mysql-operator/pkg/mysql" "github.com/percona/percona-server-mysql-operator/pkg/orchestrator" + "github.com/percona/percona-server-mysql-operator/pkg/pitr" "github.com/percona/percona-server-mysql-operator/pkg/platform" "github.com/percona/percona-server-mysql-operator/pkg/router" "github.com/percona/percona-server-mysql-operator/pkg/xtrabackup" @@ -96,7 +98,7 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req err := k8sretry.OnError(k8sretry.DefaultRetry, retriable, func() error { cr := &apiv1.PerconaServerMySQLRestore{} if err := r.Get(ctx, req.NamespacedName, cr); err != nil { - return errors.Wrapf(err, "get %v", req.NamespacedName.String()) + return errors.Wrapf(err, "get %v", req.String()) } cr.Status = status @@ -232,7 +234,15 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req case batchv1.JobFailed: status.State = apiv1.RestoreFailed case batchv1.JobComplete: - status.State = apiv1.RestoreSucceeded + if cr.Spec.PITR != nil { + pitrState, err := r.reconcilePITRJob(ctx, cr, cluster) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "reconcile pitr job") + } + status.State = pitrState + } else { + status.State = apiv1.RestoreSucceeded + } } } case apiv1.RestoreFailed, apiv1.RestoreSucceeded: @@ -256,6 +266,65 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req return ctrl.Result{}, nil } +func (r *PerconaServerMySQLRestoreReconciler) reconcilePITRJob( + ctx context.Context, + cr *apiv1.PerconaServerMySQLRestore, + cluster *apiv1.PerconaServerMySQL, +) (apiv1.RestoreState, error) { + log := logf.FromContext(ctx) + + pitrJob := &batchv1.Job{} + nn := types.NamespacedName{Name: pitr.JobName(cluster, cr), Namespace: cr.Namespace} + err := r.Get(ctx, nn, pitrJob) + if err != nil { + if !k8serrors.IsNotFound(err) { + return "", errors.Wrapf(err, "get pitr job %s", nn) + } + + log.Info("Creating PITR restore job", "jobName", nn.Name) + + bcp, err := getBackup(ctx, r.Client, cr, cluster) + if err != nil { + return "", errors.Wrap(err, "get backup") + } + + initImage, err := k8s.InitImage(ctx, r.Client, cluster, cluster.Spec.Backup) + if err != nil { + return "", errors.Wrap(err, "get operator image") + } + + job := pitr.RestoreJob(cluster, cr, bcp.Status.Storage, initImage) + if err := controllerutil.SetControllerReference(cr, job, r.Scheme); err != nil { + return "", errors.Wrapf(err, "set controller reference to Job %s/%s", job.Namespace, job.Name) + } + + if err := r.Create(ctx, job); err != nil { + return "", errors.Wrapf(err, "create pitr job %s/%s", job.Namespace, job.Name) + } + + return apiv1.RestoreRunning, nil + } + + if pitrJob.Status.Active > 0 { + return apiv1.RestoreRunning, nil + } + + for _, cond := range pitrJob.Status.Conditions { + if cond.Status != corev1.ConditionTrue { + continue + } + + switch cond.Type { + case batchv1.JobFailed: + return apiv1.RestoreFailed, nil + case batchv1.JobComplete: + return apiv1.RestoreSucceeded, nil + } + } + + return apiv1.RestoreRunning, nil +} + func (r *PerconaServerMySQLRestoreReconciler) deletePVCs(ctx context.Context, cluster *apiv1.PerconaServerMySQL) error { log := logf.FromContext(ctx) @@ -378,13 +447,13 @@ func (r *PerconaServerMySQLRestoreReconciler) unpauseCluster(ctx context.Context return k8sretry.RetryOnConflict(k8sretry.DefaultRetry, func() error { c := &apiv1.PerconaServerMySQL{} nn := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} - if err := r.Client.Get(ctx, nn, c); err != nil { + if err := r.Get(ctx, nn, c); err != nil { return err } c.Spec.Pause = false - if err := r.Client.Patch(ctx, c, client.MergeFrom(cluster)); err != nil { + if err := r.Patch(ctx, c, client.MergeFrom(cluster)); err != nil { return err } From b0795ec0be5bcc7076bbb725e4ce5c3b043d0ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 13:42:37 +0300 Subject: [PATCH 006/102] pass binlog list to pitr job --- cmd/manager/main.go | 1 + pkg/controller/psrestore/controller.go | 53 +++++++++++++++++++ pkg/pitr/pitr.go | 70 +++++++++++++++++++++++--- 3 files changed, 117 insertions(+), 7 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 0728bc5b2..adeca6db3 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -177,6 +177,7 @@ func main() { Client: nsClient, Scheme: mgr.GetScheme(), ServerVersion: serverVersion, + ClientCmd: cliCmd, NewStorageClient: storage.NewClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "PerconaServerMySQLRestore") diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index f475b0bcb..ffe5888ff 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -38,6 +38,8 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" + "github.com/percona/percona-server-mysql-operator/pkg/clientcmd" "github.com/percona/percona-server-mysql-operator/pkg/haproxy" "github.com/percona/percona-server-mysql-operator/pkg/k8s" "github.com/percona/percona-server-mysql-operator/pkg/mysql" @@ -54,6 +56,7 @@ type PerconaServerMySQLRestoreReconciler struct { client.Client Scheme *runtime.Scheme ServerVersion *platform.ServerVersion + ClientCmd clientcmd.Client NewStorageClient storage.NewClientFunc sm sync.Map @@ -283,6 +286,25 @@ func (r *PerconaServerMySQLRestoreReconciler) reconcilePITRJob( log.Info("Creating PITR restore job", "jobName", nn.Name) + binlogs, err := r.searchBinlogs(ctx, cr, cluster) + if err != nil { + return "", errors.Wrap(err, "search binlogs") + } + if len(binlogs) == 0 { + return "", errors.New("no binlogs found for the given PITR target") + } + + cm, err := pitr.BinlogsConfigMap(cluster, cr, binlogs) + if err != nil { + return "", errors.Wrap(err, "create binlogs configmap") + } + if err := controllerutil.SetControllerReference(cr, cm, r.Scheme); err != nil { + return "", errors.Wrapf(err, "set controller reference to ConfigMap %s/%s", cm.Namespace, cm.Name) + } + if err := r.Create(ctx, cm); err != nil { + return "", errors.Wrapf(err, "create binlogs configmap %s/%s", cm.Namespace, cm.Name) + } + bcp, err := getBackup(ctx, r.Client, cr, cluster) if err != nil { return "", errors.Wrap(err, "get backup") @@ -325,6 +347,37 @@ func (r *PerconaServerMySQLRestoreReconciler) reconcilePITRJob( return apiv1.RestoreRunning, nil } +func (r *PerconaServerMySQLRestoreReconciler) searchBinlogs( + ctx context.Context, + cr *apiv1.PerconaServerMySQLRestore, + cluster *apiv1.PerconaServerMySQL, +) ([]binlogserver.BinlogEntry, error) { + if cr.Spec.PITR == nil { + return nil, errors.New("pitr spec is not set") + } + + var resp *binlogserver.SearchResponse + var err error + + switch cr.Spec.PITR.Type { + case apiv1.PITRDate: + resp, err = binlogserver.SearchByTimestamp(ctx, r.Client, r.ClientCmd, cluster, cr.Spec.PITR.Date) + case apiv1.PITRGtid: + resp, err = binlogserver.SearchByGTID(ctx, r.Client, r.ClientCmd, cluster, cr.Spec.PITR.GTID) + default: + return nil, errors.Errorf("unknown PITR type: %s", cr.Spec.PITR.Type) + } + if err != nil { + return nil, errors.Wrap(err, "search binlogs") + } + + if resp.Status != "success" { + return nil, errors.Errorf("binlog search failed with status: %s", resp.Status) + } + + return resp.Result, nil +} + func (r *PerconaServerMySQLRestoreReconciler) deletePVCs(ctx context.Context, cluster *apiv1.PerconaServerMySQL) error { log := logf.FromContext(ctx) diff --git a/pkg/pitr/pitr.go b/pkg/pitr/pitr.go index d7b5507a6..61ec9c345 100644 --- a/pkg/pitr/pitr.go +++ b/pkg/pitr/pitr.go @@ -1,6 +1,7 @@ package pitr import ( + "encoding/json" "fmt" batchv1 "k8s.io/api/batch/v1" @@ -9,6 +10,7 @@ import ( "k8s.io/utils/ptr" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" "github.com/percona/percona-server-mysql-operator/pkg/k8s" "github.com/percona/percona-server-mysql-operator/pkg/mysql" "github.com/percona/percona-server-mysql-operator/pkg/naming" @@ -17,19 +19,55 @@ import ( ) const ( - appName = "pitr" - dataVolumeName = "datadir" - dataMountPath = "/var/lib/mysql" - credsVolumeName = "users" - credsMountPath = "/etc/mysql/mysql-users-secret" - tlsVolumeName = "tls" - tlsMountPath = "/etc/mysql/mysql-tls-secret" + appName = "pitr" + dataVolumeName = "datadir" + dataMountPath = "/var/lib/mysql" + credsVolumeName = "users" + credsMountPath = "/etc/mysql/mysql-users-secret" + tlsVolumeName = "tls" + tlsMountPath = "/etc/mysql/mysql-tls-secret" + binlogsVolumeName = "binlogs" + binlogsMountPath = "/etc/pitr" + binlogsKey = "binlogs.json" ) func JobName(cluster *apiv1.PerconaServerMySQL, restore *apiv1.PerconaServerMySQLRestore) string { return fmt.Sprintf("pitr-restore-%s", restore.Name) } +func BinlogsConfigMapName(restore *apiv1.PerconaServerMySQLRestore) string { + return fmt.Sprintf("pitr-binlogs-%s", restore.Name) +} + +func BinlogsConfigMap( + cluster *apiv1.PerconaServerMySQL, + restore *apiv1.PerconaServerMySQLRestore, + binlogs []binlogserver.BinlogEntry, +) (*corev1.ConfigMap, error) { + data, err := json.Marshal(binlogs) + if err != nil { + return nil, fmt.Errorf("marshal binlog entries: %w", err) + } + + labels := util.SSMapMerge(cluster.GlobalLabels(), restore.Labels(appName, naming.ComponentPITR)) + + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: BinlogsConfigMapName(restore), + Namespace: cluster.Namespace, + Labels: labels, + Annotations: cluster.GlobalAnnotations(), + }, + Data: map[string]string{ + binlogsKey: string(data), + }, + }, nil +} + func RestoreJob( cluster *apiv1.PerconaServerMySQL, restore *apiv1.PerconaServerMySQLRestore, @@ -130,6 +168,16 @@ func RestoreJob( }, }, }, + { + Name: binlogsVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: BinlogsConfigMapName(restore), + }, + }, + }, + }, }, }, }, @@ -150,6 +198,10 @@ func restoreContainer( Name: "RESTORE_NAME", Value: restore.Name, }, + { + Name: "BINLOGS_PATH", + Value: fmt.Sprintf("%s/%s", binlogsMountPath, binlogsKey), + }, } if restore.Spec.PITR != nil { @@ -230,6 +282,10 @@ func restoreContainer( Name: tlsVolumeName, MountPath: tlsMountPath, }, + { + Name: binlogsVolumeName, + MountPath: binlogsMountPath, + }, }, Command: []string{"/opt/percona/run-pitr-restore.sh"}, TerminationMessagePath: "/dev/termination-log", From aeb61df9523abb7963bc3ff15f3c211b97ddb0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 13:45:37 +0300 Subject: [PATCH 007/102] add pitr entrypoint --- build/ps-init-entrypoint.sh | 1 + build/run-pitr-restore.sh | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 build/run-pitr-restore.sh diff --git a/build/ps-init-entrypoint.sh b/build/ps-init-entrypoint.sh index 0d86d21f6..2de150010 100755 --- a/build/ps-init-entrypoint.sh +++ b/build/ps-init-entrypoint.sh @@ -40,3 +40,4 @@ install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy-global.cf install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/pmm-prerun.sh" "${BINDIR}/pmm-prerun.sh" install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/binlog-server-entrypoint.sh" "${BINDIR}/binlog-server-entrypoint.sh" +install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/run-pitr-restore.sh" "${BINDIR}/run-pitr-restore.sh" diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh new file mode 100644 index 000000000..ee9e034c7 --- /dev/null +++ b/build/run-pitr-restore.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +sleep infinity From 2930942c40caf5844afd187d6d2ac370db109afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 13:51:58 +0300 Subject: [PATCH 008/102] clean up binlog server if cluster is paused --- api/v1/perconaservermysql_types.go | 1 + pkg/controller/ps/controller.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index f69d61dde..9a20f1349 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -970,6 +970,7 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion cr.Spec.Orchestrator.Size = 0 cr.Spec.Proxy.Router.Size = 0 cr.Spec.Proxy.HAProxy.Size = 0 + cr.Spec.Backup.PiTR.Enabled = false } if cr.Spec.SecretsName == "" { diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index 7d330a3bf..e3ee3128a 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1402,6 +1402,27 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context return nil } +func (r *PerconaServerMySQLReconciler) cleanupBinlogServer(ctx context.Context, cr *apiv1.PerconaServerMySQL) error { + if cr.Spec.Backup.PiTR.Enabled { + return nil + } + + if err := r.Delete(ctx, binlogserver.StatefulSet(cr, "", "")); err != nil && !k8serrors.IsNotFound(err) { + return errors.Wrap(err, "failed to delete binlog server statefulset") + } + + if err := r.Delete(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: binlogserver.ConfigSecretName(cr), + Namespace: cr.Namespace, + }, + }); err != nil && !k8serrors.IsNotFound(err) { + return errors.Wrap(err, "failed to delete binlog server config secret") + } + + return nil +} + func (r *PerconaServerMySQLReconciler) cleanupOutdated(ctx context.Context, cr *apiv1.PerconaServerMySQL) error { if err := r.cleanupMysql(ctx, cr); err != nil { return errors.Wrap(err, "cleanup mysql") @@ -1415,6 +1436,10 @@ func (r *PerconaServerMySQLReconciler) cleanupOutdated(ctx context.Context, cr * return errors.Wrap(err, "cleanup proxies") } + if err := r.cleanupBinlogServer(ctx, cr); err != nil { + return errors.Wrap(err, "cleanup binlog server") + } + return nil } From 35558b4f8ac0085166057962996ba1e4b1bec825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 16:27:44 +0300 Subject: [PATCH 009/102] implement pitr cli --- api/v1/zz_generated.deepcopy.go | 20 +++ build/Dockerfile | 7 + build/ps-init-entrypoint.sh | 1 + build/run-pitr-restore.sh | 8 +- cmd/internal/db/db.go | 61 ++++++++ cmd/pitr/main.go | 252 ++++++++++++++++++++++++++++++++ 6 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 cmd/pitr/main.go diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index e78e2e2ad..ca874fc37 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -801,6 +801,11 @@ func (in *PerconaServerMySQLRestoreSpec) DeepCopyInto(out *PerconaServerMySQLRes *out = new(BackupContainerOptions) (*in).DeepCopyInto(*out) } + if in.PITR != nil { + in, out := &in.PITR, &out.PITR + *out = new(RestorePITRSpec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PerconaServerMySQLRestoreSpec. @@ -1090,6 +1095,21 @@ func (in *ProxySpec) DeepCopy() *ProxySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestorePITRSpec) DeepCopyInto(out *RestorePITRSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestorePITRSpec. +func (in *RestorePITRSpec) DeepCopy() *RestorePITRSpec { + if in == nil { + return nil + } + out := new(RestorePITRSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceExpose) DeepCopyInto(out *ServiceExpose) { *out = *in diff --git a/build/Dockerfile b/build/Dockerfile index 802de4ca0..11b432e33 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -50,6 +50,11 @@ RUN GOOS=$GOOS GOARCH=$TARGETARCH CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFLAG -o build/_output/bin/mysql-state-monitor \ cmd/mysql-state-monitor/main.go \ && cp -r build/_output/bin/mysql-state-monitor /usr/local/bin/mysql-state-monitor +RUN GOOS=$GOOS GOARCH=$TARGETARCH CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFLAGS \ + go build -ldflags "-w -s -X main.GitCommit=$GIT_COMMIT -X main.GitBranch=$GIT_BRANCH -X main.BuildTime=$BUILD_TIME" \ + -o build/_output/bin/pitr \ + ./cmd/pitr/ \ + && cp -r build/_output/bin/pitr /usr/local/bin/pitr FROM redhat/ubi9-minimal AS ubi9 RUN microdnf -y update && microdnf clean all @@ -91,5 +96,7 @@ COPY build/haproxy.cfg /opt/percona-server-mysql-operator/haproxy.cfg COPY build/haproxy-global.cfg /opt/percona-server-mysql-operator/haproxy-global.cfg COPY build/pmm-prerun.sh /opt/percona-server-mysql-operator/pmm-prerun.sh COPY build/binlog-server-entrypoint.sh /opt/percona-server-mysql-operator/binlog-server-entrypoint.sh +COPY --from=go_builder /usr/local/bin/pitr /opt/percona-server-mysql-operator/pitr +COPY build/run-pitr-restore.sh /opt/percona-server-mysql-operator/run-pitr-restore.sh USER 2 diff --git a/build/ps-init-entrypoint.sh b/build/ps-init-entrypoint.sh index 2de150010..ed921426a 100755 --- a/build/ps-init-entrypoint.sh +++ b/build/ps-init-entrypoint.sh @@ -40,4 +40,5 @@ install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy-global.cf install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/pmm-prerun.sh" "${BINDIR}/pmm-prerun.sh" install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/binlog-server-entrypoint.sh" "${BINDIR}/binlog-server-entrypoint.sh" +install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/pitr" "${BINDIR}/pitr" install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/run-pitr-restore.sh" "${BINDIR}/run-pitr-restore.sh" diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh index ee9e034c7..e7fb19f9d 100644 --- a/build/run-pitr-restore.sh +++ b/build/run-pitr-restore.sh @@ -1,3 +1,9 @@ #!/bin/bash -sleep infinity +set -e + +/opt/percona/pitr setup +mysqld --skip-replica-start --user=mysql & +until mysqladmin ping --silent 2>/dev/null; do sleep 1; done +/opt/percona/pitr apply +mysqladmin shutdown diff --git a/cmd/internal/db/db.go b/cmd/internal/db/db.go index 6b3c60c9e..dfa48f82a 100644 --- a/cmd/internal/db/db.go +++ b/cmd/internal/db/db.go @@ -380,3 +380,64 @@ func (d *DB) EnableSuperReadonly(ctx context.Context) error { _, err := d.db.ExecContext(ctx, "SET GLOBAL SUPER_READ_ONLY=1") return errors.Wrap(err, "set global super_read_only param to 1") } + +func (d *DB) ChangeReplicationSourceRelay(ctx context.Context, relayLogFile string, relayLogPos int) error { + _, err := d.db.ExecContext(ctx, fmt.Sprintf( + "CHANGE REPLICATION SOURCE TO RELAY_LOG_FILE='%s', RELAY_LOG_POS=%d, SOURCE_HOST='dummy'", + relayLogFile, relayLogPos)) + return errors.Wrap(err, "change replication source to relay log") +} + +func (d *DB) StartReplicaUntilGTID(ctx context.Context, gtid string) error { + _, err := d.db.ExecContext(ctx, fmt.Sprintf( + "START REPLICA SQL_THREAD UNTIL SQL_BEFORE_GTIDS='%s'", gtid)) + return errors.Wrap(err, "start replica until GTID") +} + +func (d *DB) StartReplicaUntilPosition(ctx context.Context, relayLogFile string, relayLogPos int) error { + _, err := d.db.ExecContext(ctx, fmt.Sprintf( + "START REPLICA SQL_THREAD UNTIL RELAY_LOG_FILE='%s', RELAY_LOG_POS=%d", + relayLogFile, relayLogPos)) + return errors.Wrap(err, "start replica until position") +} + +func (d *DB) WaitReplicaSQLThreadStop(ctx context.Context, pollInterval time.Duration) error { + for { + var serviceState string + err := d.db.QueryRowContext(ctx, + "SELECT SERVICE_STATE FROM replication_applier_status WHERE CHANNEL_NAME=''").Scan(&serviceState) + if err != nil { + return errors.Wrap(err, "query replication applier status") + } + + if serviceState == "OFF" { + break + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(pollInterval): + } + } + + rows, err := d.db.QueryContext(ctx, + "SELECT LAST_ERROR_NUMBER, LAST_ERROR_MESSAGE FROM replication_applier_status_by_worker WHERE CHANNEL_NAME=''") + if err != nil { + return errors.Wrap(err, "query replication applier worker status") + } + defer rows.Close() + + for rows.Next() { + var errNum int + var errMsg string + if err := rows.Scan(&errNum, &errMsg); err != nil { + return errors.Wrap(err, "scan worker status") + } + if errNum != 0 { + return errors.Errorf("replication worker error %d: %s", errNum, errMsg) + } + } + + return rows.Err() +} diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go new file mode 100644 index 000000000..ecc4b87c2 --- /dev/null +++ b/cmd/pitr/main.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" + + apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/cmd/bootstrap/utils" + "github.com/percona/percona-server-mysql-operator/cmd/internal/db" + "github.com/percona/percona-server-mysql-operator/pkg/xtrabackup/storage" +) + +type BinlogEntry struct { + Name string `json:"name"` + Size int64 `json:"size"` + URI string `json:"uri"` + PreviousGTIDs string `json:"previous_gtids"` + AddedGTIDs string `json:"added_gtids"` + MinTimestamp string `json:"min_timestamp"` + MaxTimestamp string `json:"max_timestamp"` +} + +func main() { + if len(os.Args) < 2 { + log.Fatal("usage: pitr ") + } + + ctx := context.Background() + + switch os.Args[1] { + case "setup": + if err := runSetup(ctx); err != nil { + log.Fatalf("setup failed: %v", err) + } + case "apply": + if err := runApply(ctx); err != nil { + log.Fatalf("apply failed: %v", err) + } + default: + log.Fatalf("unknown subcommand: %s", os.Args[1]) + } +} + +func runSetup(ctx context.Context) error { + binlogsPath := os.Getenv("BINLOGS_PATH") + if binlogsPath == "" { + return fmt.Errorf("BINLOGS_PATH is not set") + } + + data, err := os.ReadFile(binlogsPath) + if err != nil { + return fmt.Errorf("read binlogs file: %w", err) + } + + var entries []BinlogEntry + if err := json.Unmarshal(data, &entries); err != nil { + return fmt.Errorf("parse binlogs json: %w", err) + } + + if len(entries) == 0 { + return fmt.Errorf("no binlog entries found") + } + + hostname, err := os.Hostname() + if err != nil { + return fmt.Errorf("get hostname: %w", err) + } + + storageType := os.Getenv("STORAGE_TYPE") + + var s3Client storage.Storage + switch storageType { + case string(apiv1.BackupStorageS3), string(apiv1.BackupStorageGCS): + endpoint := os.Getenv("AWS_ENDPOINT") + accessKey := os.Getenv("AWS_ACCESS_KEY_ID") + secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") + region := os.Getenv("AWS_DEFAULT_REGION") + bucket := os.Getenv("S3_BUCKET") + verifyTLS := os.Getenv("VERIFY_TLS") != "false" + + if storageType == string(apiv1.BackupStorageGCS) { + s3Client, err = storage.NewGCS(ctx, endpoint, accessKey, secretKey, bucket, "", verifyTLS) + } else { + s3Client, err = storage.NewS3(ctx, endpoint, accessKey, secretKey, bucket, "", region, verifyTLS) + } + if err != nil { + return fmt.Errorf("create S3 client: %w", err) + } + case string(apiv1.BackupStorageAzure): + storageAccount := os.Getenv("AZURE_STORAGE_ACCOUNT") + accessKey := os.Getenv("AZURE_ACCESS_KEY") + endpoint := os.Getenv("AZURE_ENDPOINT") + container := os.Getenv("AZURE_CONTAINER") + + s3Client, err = storage.NewAzure(ctx, storageAccount, accessKey, endpoint, container, "") + if err != nil { + return fmt.Errorf("create Azure client: %w", err) + } + default: + return fmt.Errorf("unsupported storage type: %s", storageType) + } + + var relayLogFiles []string + for i, entry := range entries { + relayLogName := fmt.Sprintf("%s-relay-bin.%06d", hostname, i+1) + relayLogPath := fmt.Sprintf("/var/lib/mysql/%s", relayLogName) + + log.Printf("downloading binlog %s to %s", entry.URI, relayLogPath) + + obj, err := s3Client.GetObject(ctx, entry.URI) + if err != nil { + return fmt.Errorf("download binlog %s: %w", entry.URI, err) + } + + f, err := os.Create(relayLogPath) + if err != nil { + obj.Close() + return fmt.Errorf("create relay log file %s: %w", relayLogPath, err) + } + + _, err = io.Copy(f, obj) + obj.Close() + f.Close() + if err != nil { + return fmt.Errorf("write relay log file %s: %w", relayLogPath, err) + } + + relayLogFiles = append(relayLogFiles, "./"+relayLogName) + } + + indexPath := fmt.Sprintf("/var/lib/mysql/%s-relay-bin.index", hostname) + indexContent := strings.Join(relayLogFiles, "\n") + "\n" + if err := os.WriteFile(indexPath, []byte(indexContent), 0644); err != nil { + return fmt.Errorf("write relay log index: %w", err) + } + + log.Printf("setup complete: %d relay log files written", len(relayLogFiles)) + return nil +} + +func runApply(ctx context.Context) error { + pitrType := os.Getenv("PITR_TYPE") + pitrDate := os.Getenv("PITR_DATE") + pitrGTID := os.Getenv("PITR_GTID") + + hostname, err := os.Hostname() + if err != nil { + return fmt.Errorf("get hostname: %w", err) + } + + operatorPass, err := utils.GetSecret(apiv1.UserOperator) + if err != nil { + return fmt.Errorf("get operator password: %w", err) + } + + database, err := db.NewDatabase(ctx, db.DBParams{ + User: apiv1.UserOperator, + Pass: operatorPass, + Host: "localhost", + }) + if err != nil { + return fmt.Errorf("connect to MySQL: %w", err) + } + defer database.Close() + + binlogsPath := os.Getenv("BINLOGS_PATH") + data, err := os.ReadFile(binlogsPath) + if err != nil { + return fmt.Errorf("read binlogs file: %w", err) + } + + var entries []BinlogEntry + if err := json.Unmarshal(data, &entries); err != nil { + return fmt.Errorf("parse binlogs json: %w", err) + } + + lastRelayLog := fmt.Sprintf("%s-relay-bin.%06d", hostname, len(entries)) + lastRelayLogPath := fmt.Sprintf("/var/lib/mysql/%s", lastRelayLog) + + var stopPos int + if pitrType == "date" { + stopPos, err = getStopPosition(lastRelayLogPath, pitrDate) + if err != nil { + return fmt.Errorf("get stop position: %w", err) + } + log.Printf("stop position for date %s: %d", pitrDate, stopPos) + } + + firstRelayLog := fmt.Sprintf("%s-relay-bin.000001", hostname) + + if err := database.ChangeReplicationSourceRelay(ctx, firstRelayLog, 1); err != nil { + return fmt.Errorf("change replication source: %w", err) + } + + switch pitrType { + case "gtid": + log.Printf("starting replica until GTID: %s", pitrGTID) + if err := database.StartReplicaUntilGTID(ctx, pitrGTID); err != nil { + return fmt.Errorf("start replica until GTID: %w", err) + } + case "date": + log.Printf("starting replica until position: %s:%d", lastRelayLog, stopPos) + if err := database.StartReplicaUntilPosition(ctx, lastRelayLog, stopPos); err != nil { + return fmt.Errorf("start replica until position: %w", err) + } + default: + return fmt.Errorf("unsupported PITR type: %s", pitrType) + } + + log.Println("waiting for replication to complete...") + if err := database.WaitReplicaSQLThreadStop(ctx, time.Second); err != nil { + return fmt.Errorf("wait for replication: %w", err) + } + + if err := database.ResetReplication(ctx); err != nil { + return fmt.Errorf("reset replication: %w", err) + } + + log.Println("PITR apply complete") + return nil +} + +func getStopPosition(relayLogPath, stopDatetime string) (int, error) { + cmd := exec.Command("mysqlbinlog", "--stop-datetime="+stopDatetime, relayLogPath) + out, err := cmd.Output() + if err != nil { + return 0, fmt.Errorf("run mysqlbinlog: %w", err) + } + + re := regexp.MustCompile(`end_log_pos (\d+)`) + matches := re.FindAllStringSubmatch(string(out), -1) + if len(matches) == 0 { + return 0, fmt.Errorf("no end_log_pos found in mysqlbinlog output") + } + + lastMatch := matches[len(matches)-1] + pos, err := strconv.Atoi(lastMatch[1]) + if err != nil { + return 0, fmt.Errorf("parse end_log_pos: %w", err) + } + + return pos, nil +} From de34a010ff4457945d4126880ea46313d8a68bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 16:42:47 +0300 Subject: [PATCH 010/102] fix s3 uri --- pkg/controller/ps/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index e3ee3128a..dbd47075b 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1320,7 +1320,7 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context accessKey := s3Secret.Data[secret.CredentialsAWSAccessKey] secretKey := s3Secret.Data[secret.CredentialsAWSSecretKey] - s3Uri := fmt.Sprintf("https://%s:%s@%s", accessKey, secretKey, s3.EndpointURL) + s3Uri := fmt.Sprintf("https://%s:%s@%s/%s", accessKey, secretKey, s3.EndpointURL, s3.Bucket) if len(s3.Prefix) > 0 { s3Uri += fmt.Sprintf("/%s", s3.Prefix) } From 8b389645fb58139f45926be884b3b87ad2ba714f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 17:53:37 +0300 Subject: [PATCH 011/102] fix pitr job --- pkg/controller/psrestore/controller.go | 61 ++++++++++++++++++-------- pkg/pitr/pitr.go | 26 +++-------- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index ffe5888ff..c75164141 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -18,6 +18,7 @@ package psrestore import ( "context" + "encoding/json" "fmt" "sync" "time" @@ -178,6 +179,12 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req } defer r.sm.Delete(cr.Spec.ClusterName) + if cr.Spec.PITR != nil { + if err := r.reconcilePITRConfig(ctx, cr, cluster); err != nil { + return ctrl.Result{}, errors.Wrap(err, "reconcile pitr config") + } + } + log.Info("Pausing cluster", "cluster", cluster.Name) if err := r.pauseCluster(ctx, cluster); err != nil { if errors.Is(err, ErrWaitingTermination) { @@ -269,6 +276,41 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req return ctrl.Result{}, nil } +func (r *PerconaServerMySQLRestoreReconciler) reconcilePITRConfig( + ctx context.Context, + cr *apiv1.PerconaServerMySQLRestore, + cluster *apiv1.PerconaServerMySQL, +) error { + cm := pitr.BinlogsConfigMap(cluster, cr) + if err := r.Get(ctx, client.ObjectKeyFromObject(cm), new(corev1.ConfigMap)); err == nil { + return nil + } + + binlogs, err := r.searchBinlogs(ctx, cr, cluster) + if err != nil { + return errors.Wrap(err, "search binlogs") + } + if len(binlogs) == 0 { + return errors.New("no binlogs found for the given PITR target") + } + + data, err := json.Marshal(binlogs) + if err != nil { + return errors.Wrap(err, "marshal binlog entries") + } + + cm.Data[pitr.BinlogsConfigKey] = string(data) + + if err := controllerutil.SetControllerReference(cr, cm, r.Scheme); err != nil { + return errors.Wrapf(err, "set controller reference to ConfigMap %s/%s", cm.Namespace, cm.Name) + } + if err := r.Create(ctx, cm); err != nil { + return errors.Wrapf(err, "create binlogs configmap %s/%s", cm.Namespace, cm.Name) + } + + return nil +} + func (r *PerconaServerMySQLRestoreReconciler) reconcilePITRJob( ctx context.Context, cr *apiv1.PerconaServerMySQLRestore, @@ -286,25 +328,6 @@ func (r *PerconaServerMySQLRestoreReconciler) reconcilePITRJob( log.Info("Creating PITR restore job", "jobName", nn.Name) - binlogs, err := r.searchBinlogs(ctx, cr, cluster) - if err != nil { - return "", errors.Wrap(err, "search binlogs") - } - if len(binlogs) == 0 { - return "", errors.New("no binlogs found for the given PITR target") - } - - cm, err := pitr.BinlogsConfigMap(cluster, cr, binlogs) - if err != nil { - return "", errors.Wrap(err, "create binlogs configmap") - } - if err := controllerutil.SetControllerReference(cr, cm, r.Scheme); err != nil { - return "", errors.Wrapf(err, "set controller reference to ConfigMap %s/%s", cm.Namespace, cm.Name) - } - if err := r.Create(ctx, cm); err != nil { - return "", errors.Wrapf(err, "create binlogs configmap %s/%s", cm.Namespace, cm.Name) - } - bcp, err := getBackup(ctx, r.Client, cr, cluster) if err != nil { return "", errors.Wrap(err, "get backup") diff --git a/pkg/pitr/pitr.go b/pkg/pitr/pitr.go index 61ec9c345..0f585a8a6 100644 --- a/pkg/pitr/pitr.go +++ b/pkg/pitr/pitr.go @@ -1,7 +1,6 @@ package pitr import ( - "encoding/json" "fmt" batchv1 "k8s.io/api/batch/v1" @@ -10,7 +9,6 @@ import ( "k8s.io/utils/ptr" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" - "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" "github.com/percona/percona-server-mysql-operator/pkg/k8s" "github.com/percona/percona-server-mysql-operator/pkg/mysql" "github.com/percona/percona-server-mysql-operator/pkg/naming" @@ -28,7 +26,7 @@ const ( tlsMountPath = "/etc/mysql/mysql-tls-secret" binlogsVolumeName = "binlogs" binlogsMountPath = "/etc/pitr" - binlogsKey = "binlogs.json" + BinlogsConfigKey = "binlogs.json" ) func JobName(cluster *apiv1.PerconaServerMySQL, restore *apiv1.PerconaServerMySQLRestore) string { @@ -39,16 +37,7 @@ func BinlogsConfigMapName(restore *apiv1.PerconaServerMySQLRestore) string { return fmt.Sprintf("pitr-binlogs-%s", restore.Name) } -func BinlogsConfigMap( - cluster *apiv1.PerconaServerMySQL, - restore *apiv1.PerconaServerMySQLRestore, - binlogs []binlogserver.BinlogEntry, -) (*corev1.ConfigMap, error) { - data, err := json.Marshal(binlogs) - if err != nil { - return nil, fmt.Errorf("marshal binlog entries: %w", err) - } - +func BinlogsConfigMap(cluster *apiv1.PerconaServerMySQL, restore *apiv1.PerconaServerMySQLRestore) *corev1.ConfigMap { labels := util.SSMapMerge(cluster.GlobalLabels(), restore.Labels(appName, naming.ComponentPITR)) return &corev1.ConfigMap{ @@ -62,10 +51,7 @@ func BinlogsConfigMap( Labels: labels, Annotations: cluster.GlobalAnnotations(), }, - Data: map[string]string{ - binlogsKey: string(data), - }, - }, nil + } } func RestoreJob( @@ -200,7 +186,7 @@ func restoreContainer( }, { Name: "BINLOGS_PATH", - Value: fmt.Sprintf("%s/%s", binlogsMountPath, binlogsKey), + Value: fmt.Sprintf("%s/%s", binlogsMountPath, BinlogsConfigKey), }, } @@ -262,8 +248,8 @@ func restoreContainer( return corev1.Container{ Name: appName, - Image: binlogServer.Image, - ImagePullPolicy: binlogServer.ImagePullPolicy, + Image: cluster.Spec.MySQL.Image, + ImagePullPolicy: cluster.Spec.MySQL.ImagePullPolicy, Env: envs, VolumeMounts: []corev1.VolumeMount{ { From 0712c3aa5b663072c858eb2c25bb4e12be53a5c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 19 Mar 2026 22:36:44 +0300 Subject: [PATCH 012/102] fix apply phase --- build/run-pitr-restore.sh | 22 +++++++-- cmd/internal/db/db.go | 5 ++ cmd/pitr/main.go | 97 +++++++++++++++++++-------------------- 3 files changed, 72 insertions(+), 52 deletions(-) diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh index e7fb19f9d..6624fc638 100644 --- a/build/run-pitr-restore.sh +++ b/build/run-pitr-restore.sh @@ -2,8 +2,24 @@ set -e +# TODO: Add support for data at rest encryption + +echo "Starting mysqld" +mysqld \ + --admin-address=$(hostname -I) \ + --skip-replica-start \ + --user=mysql \ + --read-only=ON \ + --super-read-only=ON \ + --gtid-mode=ON \ + --enforce-gtid-consistency=ON >/tmp/mysqld.log 2>&1 & + +until mysqladmin -u operator -p$(/dev/null; do + sleep 1; +done + /opt/percona/pitr setup -mysqld --skip-replica-start --user=mysql & -until mysqladmin ping --silent 2>/dev/null; do sleep 1; done /opt/percona/pitr apply -mysqladmin shutdown + +echo "Stopping mysqld" +mysqladmin -u operator -p$(") @@ -61,7 +54,7 @@ func runSetup(ctx context.Context) error { return fmt.Errorf("read binlogs file: %w", err) } - var entries []BinlogEntry + var entries []binlogserver.BinlogEntry if err := json.Unmarshal(data, &entries); err != nil { return fmt.Errorf("parse binlogs json: %w", err) } @@ -75,38 +68,16 @@ func runSetup(ctx context.Context) error { return fmt.Errorf("get hostname: %w", err) } - storageType := os.Getenv("STORAGE_TYPE") - - var s3Client storage.Storage - switch storageType { - case string(apiv1.BackupStorageS3), string(apiv1.BackupStorageGCS): - endpoint := os.Getenv("AWS_ENDPOINT") - accessKey := os.Getenv("AWS_ACCESS_KEY_ID") - secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") - region := os.Getenv("AWS_DEFAULT_REGION") - bucket := os.Getenv("S3_BUCKET") - verifyTLS := os.Getenv("VERIFY_TLS") != "false" + endpoint := os.Getenv("AWS_ENDPOINT") + accessKey := os.Getenv("AWS_ACCESS_KEY_ID") + secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") + region := os.Getenv("AWS_DEFAULT_REGION") + bucket := os.Getenv("S3_BUCKET") + verifyTLS := os.Getenv("VERIFY_TLS") != "false" - if storageType == string(apiv1.BackupStorageGCS) { - s3Client, err = storage.NewGCS(ctx, endpoint, accessKey, secretKey, bucket, "", verifyTLS) - } else { - s3Client, err = storage.NewS3(ctx, endpoint, accessKey, secretKey, bucket, "", region, verifyTLS) - } - if err != nil { - return fmt.Errorf("create S3 client: %w", err) - } - case string(apiv1.BackupStorageAzure): - storageAccount := os.Getenv("AZURE_STORAGE_ACCOUNT") - accessKey := os.Getenv("AZURE_ACCESS_KEY") - endpoint := os.Getenv("AZURE_ENDPOINT") - container := os.Getenv("AZURE_CONTAINER") - - s3Client, err = storage.NewAzure(ctx, storageAccount, accessKey, endpoint, container, "") - if err != nil { - return fmt.Errorf("create Azure client: %w", err) - } - default: - return fmt.Errorf("unsupported storage type: %s", storageType) + s3Client, err := storage.NewS3(ctx, endpoint, accessKey, secretKey, bucket, "", region, verifyTLS) + if err != nil { + return fmt.Errorf("create S3 client: %w", err) } var relayLogFiles []string @@ -114,9 +85,14 @@ func runSetup(ctx context.Context) error { relayLogName := fmt.Sprintf("%s-relay-bin.%06d", hostname, i+1) relayLogPath := fmt.Sprintf("/var/lib/mysql/%s", relayLogName) - log.Printf("downloading binlog %s to %s", entry.URI, relayLogPath) + objectKey, err := objectKeyFromURI(entry.URI) + if err != nil { + return fmt.Errorf("parse URI %s: %w", entry.URI, err) + } + + log.Printf("downloading binlog %s to %s", objectKey, relayLogPath) - obj, err := s3Client.GetObject(ctx, entry.URI) + obj, err := s3Client.GetObject(ctx, objectKey) if err != nil { return fmt.Errorf("download binlog %s: %w", entry.URI, err) } @@ -165,7 +141,7 @@ func runApply(ctx context.Context) error { database, err := db.NewDatabase(ctx, db.DBParams{ User: apiv1.UserOperator, Pass: operatorPass, - Host: "localhost", + Host: "127.0.0.1", }) if err != nil { return fmt.Errorf("connect to MySQL: %w", err) @@ -178,7 +154,7 @@ func runApply(ctx context.Context) error { return fmt.Errorf("read binlogs file: %w", err) } - var entries []BinlogEntry + var entries []binlogserver.BinlogEntry if err := json.Unmarshal(data, &entries); err != nil { return fmt.Errorf("parse binlogs json: %w", err) } @@ -192,12 +168,18 @@ func runApply(ctx context.Context) error { if err != nil { return fmt.Errorf("get stop position: %w", err) } - log.Printf("stop position for date %s: %d", pitrDate, stopPos) + log.Printf("stop position for date %s: %s %d", pitrDate, lastRelayLog, stopPos) } firstRelayLog := fmt.Sprintf("%s-relay-bin.000001", hostname) - if err := database.ChangeReplicationSourceRelay(ctx, firstRelayLog, 1); err != nil { + log.Println("running 'RESET BINARY LOGS AND GTIDS'") + if err := database.ResetBinaryLogAndGTIDs(ctx); err != nil { + return err + } + + log.Println("running 'CHANGE REPLICATION SOURCE'") + if err := database.ChangeReplicationSourceRelay(ctx, firstRelayLog, 4); err != nil { return fmt.Errorf("change replication source: %w", err) } @@ -221,8 +203,14 @@ func runApply(ctx context.Context) error { return fmt.Errorf("wait for replication: %w", err) } + log.Println("stopping replication") + if err := database.StopReplication(ctx); err != nil { + return errors.Wrap(err, "stop replication") + } + + log.Println("running 'RESET REPLICA ALL'") if err := database.ResetReplication(ctx); err != nil { - return fmt.Errorf("reset replication: %w", err) + return errors.Wrap(err, "reset replication") } log.Println("PITR apply complete") @@ -250,3 +238,14 @@ func getStopPosition(relayLogPath, stopDatetime string) (int, error) { return pos, nil } + +// objectKeyFromURI extracts the S3 object key from a full URI. +// e.g. "https://minio-service:9000/binlogs/binlog.000001" -> "binlogs/binlog.000001" +func objectKeyFromURI(uri string) (string, error) { + u, err := url.Parse(uri) + if err != nil { + return "", fmt.Errorf("parse URL: %w", err) + } + // Path starts with "/", trim the leading slash to get the object key + return strings.TrimPrefix(u.Path, "/"), nil +} From eb3219cfb30b4b822bbff1a3ab3f606d0b1e4ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Fri, 20 Mar 2026 11:21:59 +0300 Subject: [PATCH 013/102] allow protocol in endpoint url --- pkg/controller/ps/controller.go | 17 ++++++++++++++++- pkg/controller/psrestore/controller.go | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index dbd47075b..c8461c6e8 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1320,7 +1320,12 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context accessKey := s3Secret.Data[secret.CredentialsAWSAccessKey] secretKey := s3Secret.Data[secret.CredentialsAWSSecretKey] - s3Uri := fmt.Sprintf("https://%s:%s@%s/%s", accessKey, secretKey, s3.EndpointURL, s3.Bucket) + protocol, host, err := parseEndpointURL(s3.EndpointURL) + if err != nil { + return errors.Wrap(err, "parse endpoint URL") + } + + s3Uri := fmt.Sprintf("%s://%s:%s@%s/%s", protocol, accessKey, secretKey, host, s3.Bucket) if len(s3.Prefix) > 0 { s3Uri += fmt.Sprintf("/%s", s3.Prefix) } @@ -1402,6 +1407,16 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context return nil } +// parseEndpointURL extracts the protocol and host from an endpoint URL. +// Expected formats: "s3://s3.amazonaws.com", "https://minio-service:9000" +func parseEndpointURL(endpointURL string) (protocol, host string, err error) { + idx := strings.Index(endpointURL, "://") + if idx < 0 { + return "", "", fmt.Errorf("endpoint URL %q must include protocol (e.g. s3://... or https://...)", endpointURL) + } + return endpointURL[:idx], endpointURL[idx+3:], nil +} + func (r *PerconaServerMySQLReconciler) cleanupBinlogServer(ctx context.Context, cr *apiv1.PerconaServerMySQL) error { if cr.Spec.Backup.PiTR.Enabled { return nil diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index c75164141..8f8571f04 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -299,6 +299,7 @@ func (r *PerconaServerMySQLRestoreReconciler) reconcilePITRConfig( return errors.Wrap(err, "marshal binlog entries") } + cm.Data = make(map[string]string) cm.Data[pitr.BinlogsConfigKey] = string(data) if err := controllerutil.SetControllerReference(cr, cm, r.Scheme); err != nil { From 2e9433df1e40a463e422b5162a7ae964b78c8fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Fri, 20 Mar 2026 11:22:12 +0300 Subject: [PATCH 014/102] don't run reset --- cmd/pitr/main.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 41589ed02..10f6af26c 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -173,11 +173,6 @@ func runApply(ctx context.Context) error { firstRelayLog := fmt.Sprintf("%s-relay-bin.000001", hostname) - log.Println("running 'RESET BINARY LOGS AND GTIDS'") - if err := database.ResetBinaryLogAndGTIDs(ctx); err != nil { - return err - } - log.Println("running 'CHANGE REPLICATION SOURCE'") if err := database.ChangeReplicationSourceRelay(ctx, firstRelayLog, 4); err != nil { return fmt.Errorf("change replication source: %w", err) From cd39d9e9d5fecc4a1aef53e0ef52526db9f4f2c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Fri, 20 Mar 2026 11:22:30 +0300 Subject: [PATCH 015/102] fix admin connections --- build/run-pitr-restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh index 6624fc638..f95f239ce 100644 --- a/build/run-pitr-restore.sh +++ b/build/run-pitr-restore.sh @@ -6,7 +6,7 @@ set -e echo "Starting mysqld" mysqld \ - --admin-address=$(hostname -I) \ + --admin-address=127.0.0.1 \ --skip-replica-start \ --user=mysql \ --read-only=ON \ From 031f2504eaf2d17648673a3a82753e3c80075f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Sun, 22 Mar 2026 18:34:03 +0300 Subject: [PATCH 016/102] update manifests --- .../ps.percona.com_perconaservermysqlrestores.yaml | 11 +++++++++++ deploy/bundle.yaml | 11 +++++++++++ deploy/crd.yaml | 11 +++++++++++ deploy/cw-bundle.yaml | 11 +++++++++++ 4 files changed, 44 insertions(+) diff --git a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml index 6ea227042..8f3e987e0 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml @@ -1136,6 +1136,17 @@ spec: type: object type: array type: object + pitr: + properties: + date: + type: string + gtid: + type: string + type: + type: string + required: + - type + type: object required: - clusterName type: object diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 3939e903c..fcb9c0d56 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -2304,6 +2304,17 @@ spec: type: object type: array type: object + pitr: + properties: + date: + type: string + gtid: + type: string + type: + type: string + required: + - type + type: object required: - clusterName type: object diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 27401d4b3..df8820902 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -2304,6 +2304,17 @@ spec: type: object type: array type: object + pitr: + properties: + date: + type: string + gtid: + type: string + type: + type: string + required: + - type + type: object required: - clusterName type: object diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 312b7f789..d469cd18b 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -2304,6 +2304,17 @@ spec: type: object type: array type: object + pitr: + properties: + date: + type: string + gtid: + type: string + type: + type: string + required: + - type + type: object required: - clusterName type: object From b7d0b94e3a2c94c2148df0f7f30dec1a277120d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Sun, 22 Mar 2026 18:35:47 +0300 Subject: [PATCH 017/102] add binlog server to cr --- deploy/cr.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 51089674c..31e919374 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -663,6 +663,21 @@ spec: enabled: true pitr: enabled: false + binlogServer: + size: 1 + image: perconalab/percona-binlog-server:0.2.0 + connectTimeout: 60 + readTimeout: 60 + writeTimeout: 60 + serverId: 42 + idleTime: 60 + storage: + s3: + bucket: S3-BACKUP-BUCKET-NAME-HERE + credentialsSecret: ps-cluster1-s3-credentials + # endpointUrl: https://s3.amazonaws.com + # prefix: PREFIX_NAME + region: us-west-2 # sourcePod: ps-cluster1-mysql-1 image: perconalab/percona-server-mysql-operator:main-backup8.4 imagePullPolicy: Always From 0a6773cf54f46ceada489fe3d5c6b4f8d0661fb8 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Mon, 23 Mar 2026 17:33:43 +0200 Subject: [PATCH 018/102] add e2e test and fix status starting state --- e2e-tests/run-pr.csv | 1 + e2e-tests/tests/pitr-minio/00-assert.yaml | 9 +++ .../tests/pitr-minio/00-minio-secret.yaml | 7 ++ e2e-tests/tests/pitr-minio/01-assert.yaml | 26 +++++++ .../tests/pitr-minio/01-deploy-operator.yaml | 16 +++++ e2e-tests/tests/pitr-minio/02-assert.yaml | 72 +++++++++++++++++++ .../tests/pitr-minio/02-create-cluster.yaml | 36 ++++++++++ e2e-tests/tests/pitr-minio/03-write-data.yaml | 16 +++++ e2e-tests/tests/pitr-minio/04-assert.yaml | 12 ++++ .../tests/pitr-minio/04-create-backup.yaml | 9 +++ .../tests/pitr-minio/05-write-more-data.yaml | 12 ++++ e2e-tests/tests/pitr-minio/06-assert.yaml | 31 ++++++++ .../pitr-minio/06-create-pitr-restore.yaml | 29 ++++++++ e2e-tests/tests/pitr-minio/07-assert.yaml | 24 +++++++ e2e-tests/tests/pitr-minio/07-read-data.yaml | 16 +++++ .../tests/pitr-minio/98-drop-finalizer.yaml | 5 ++ .../99-remove-cluster-gracefully.yaml | 16 +++++ pkg/binlogserver/search.go | 2 +- pkg/controller/psrestore/controller.go | 3 + 19 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 e2e-tests/tests/pitr-minio/00-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/00-minio-secret.yaml create mode 100644 e2e-tests/tests/pitr-minio/01-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/01-deploy-operator.yaml create mode 100644 e2e-tests/tests/pitr-minio/02-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/02-create-cluster.yaml create mode 100644 e2e-tests/tests/pitr-minio/03-write-data.yaml create mode 100644 e2e-tests/tests/pitr-minio/04-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/04-create-backup.yaml create mode 100644 e2e-tests/tests/pitr-minio/05-write-more-data.yaml create mode 100644 e2e-tests/tests/pitr-minio/06-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml create mode 100644 e2e-tests/tests/pitr-minio/07-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/07-read-data.yaml create mode 100644 e2e-tests/tests/pitr-minio/98-drop-finalizer.yaml create mode 100644 e2e-tests/tests/pitr-minio/99-remove-cluster-gracefully.yaml diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 043076877..cd57880f9 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -8,6 +8,7 @@ config-router,8.0 config-router,8.4 demand-backup-minio,8.0 demand-backup-minio,8.4 +pitr-minio,8.4 demand-backup-cloud,8.4 demand-backup-retry,8.4 async-data-at-rest-encryption,8.0 diff --git a/e2e-tests/tests/pitr-minio/00-assert.yaml b/e2e-tests/tests/pitr-minio/00-assert.yaml new file mode 100644 index 000000000..fecdb222e --- /dev/null +++ b/e2e-tests/tests/pitr-minio/00-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 150 +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +type: Opaque diff --git a/e2e-tests/tests/pitr-minio/00-minio-secret.yaml b/e2e-tests/tests/pitr-minio/00-minio-secret.yaml new file mode 100644 index 000000000..3c797f054 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/00-minio-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +stringData: + AWS_ACCESS_KEY_ID: some-access$\n"-key + AWS_SECRET_ACCESS_KEY: some-$\n"secret-key diff --git a/e2e-tests/tests/pitr-minio/01-assert.yaml b/e2e-tests/tests/pitr-minio/01-assert.yaml new file mode 100644 index 000000000..5f346cb51 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/01-assert.yaml @@ -0,0 +1,26 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 150 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: perconaservermysqls.ps.percona.com +spec: + group: ps.percona.com + names: + kind: PerconaServerMySQL + listKind: PerconaServerMySQLList + plural: perconaservermysqls + shortNames: + - ps + singular: perconaservermysql + scope: Namespaced +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/pitr-minio/01-deploy-operator.yaml b/e2e-tests/tests/pitr-minio/01-deploy-operator.yaml new file mode 100644 index 000000000..c96e1f6b9 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/01-deploy-operator.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + apply_s3_storage_secrets + deploy_operator + deploy_tls_cluster_secrets + deploy_client + deploy_minio + timeout: 300 diff --git a/e2e-tests/tests/pitr-minio/02-assert.yaml b/e2e-tests/tests/pitr-minio/02-assert.yaml new file mode 100644 index 000000000..9b7df6774 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/02-assert.yaml @@ -0,0 +1,72 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: pitr-minio-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: pitr-minio-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: pitr-minio-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: pitr-minio-binlog-server +status: + observedGeneration: 1 + replicas: 1 + readyReplicas: 1 + currentReplicas: 1 + updatedReplicas: 1 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/pitr-minio/02-create-cluster.yaml b/e2e-tests/tests/pitr-minio/02-create-cluster.yaml new file mode 100644 index 000000000..384e81518 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/02-create-cluster.yaml @@ -0,0 +1,36 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval '.spec.mysql.clusterType="async"' - \ + | yq eval ".spec.mysql.size=3" - \ + | yq eval ".spec.proxy.haproxy.enabled=true" - \ + | yq eval ".spec.proxy.haproxy.size=3" - \ + | yq eval ".spec.orchestrator.enabled=true" - \ + | yq eval ".spec.orchestrator.size=3" - \ + | yq eval ".spec.backup.backoffLimit=3" - \ + | yq eval '.spec.backup.storages.minio.type="s3"' - \ + | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.minio.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.storages.minio.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.storages.minio.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.storages.minio.containerOptions.env[0].name="VERIFY_TLS"' - \ + | yq eval '.spec.backup.storages.minio.containerOptions.env[0].value="false"' - \ + | yq eval '.spec.backup.pitr.enabled=true' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.pitr.binlogServer.storage.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.pitr.binlogServer.serverId=100' - \ + | yq eval '.spec.backup.pitr.binlogServer.connectTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.readTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.writeTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.idleTime=3' - \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/pitr-minio/03-write-data.yaml b/e2e-tests/tests/pitr-minio/03-write-data.yaml new file mode 100644 index 000000000..d490db677 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/03-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ + "-h $(get_haproxy_svc $(get_cluster_name))" + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100500)" \ + "-h $(get_haproxy_svc $(get_cluster_name))" diff --git a/e2e-tests/tests/pitr-minio/04-assert.yaml b/e2e-tests/tests/pitr-minio/04-assert.yaml new file mode 100644 index 000000000..2f575b227 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/04-assert.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1 +metadata: + name: pitr-minio-backup + finalizers: + - percona.com/delete-backup +status: + state: Succeeded diff --git a/e2e-tests/tests/pitr-minio/04-create-backup.yaml b/e2e-tests/tests/pitr-minio/04-create-backup.yaml new file mode 100644 index 000000000..b1015177c --- /dev/null +++ b/e2e-tests/tests/pitr-minio/04-create-backup.yaml @@ -0,0 +1,9 @@ +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQLBackup +metadata: + name: pitr-minio-backup + finalizers: + - percona.com/delete-backup +spec: + clusterName: pitr-minio + storageName: minio diff --git a/e2e-tests/tests/pitr-minio/05-write-more-data.yaml b/e2e-tests/tests/pitr-minio/05-write-more-data.yaml new file mode 100644 index 000000000..ec30f75d5 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/05-write-more-data.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100501)" \ + "-h $(get_haproxy_svc $(get_cluster_name))" diff --git a/e2e-tests/tests/pitr-minio/06-assert.yaml b/e2e-tests/tests/pitr-minio/06-assert.yaml new file mode 100644 index 000000000..0c371a120 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/06-assert.yaml @@ -0,0 +1,31 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +kind: PerconaServerMySQLRestore +apiVersion: ps.percona.com/v1 +metadata: + name: pitr-minio-restore +status: + state: Succeeded diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml new file mode 100644 index 000000000..2d0b7c63f --- /dev/null +++ b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml @@ -0,0 +1,29 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 180 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + PITR_GTID=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_haproxy_svc ${cluster_name})" | tr -d '\n') + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100502)" \ + "-h $(get_haproxy_svc ${cluster_name})" + + sleep 60 + + echo '{}' \ + | yq eval '.apiVersion = "ps.percona.com/v1"' - \ + | yq eval '.kind = "PerconaServerMySQLRestore"' - \ + | yq eval '.metadata.name = "pitr-minio-restore"' - \ + | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ + | yq eval '.spec.backupName = "pitr-minio-backup"' - \ + | yq eval '.spec.pitr.type = "gtid"' - \ + | yq eval ".spec.pitr.gtid = \"${PITR_GTID}\"" - \ + | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/pitr-minio/07-assert.yaml b/e2e-tests/tests/pitr-minio/07-assert.yaml new file mode 100644 index 000000000..bbd6bf1cc --- /dev/null +++ b/e2e-tests/tests/pitr-minio/07-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-data-0 +data: + max_id: "100501" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-data-1 +data: + max_id: "100501" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-data-2 +data: + max_id: "100501" diff --git a/e2e-tests/tests/pitr-minio/07-read-data.yaml b/e2e-tests/tests/pitr-minio/07-read-data.yaml new file mode 100644 index 000000000..68ba007c9 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/07-read-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + for i in 0 1 2; do + max_id=$(run_mysql "SELECT MAX(id) FROM myDB.myTable" "-h ${cluster_name}-mysql-${i}.${cluster_name}-mysql") + kubectl create configmap -n "${NAMESPACE}" 07-read-data-${i} --from-literal=max_id="${max_id}" + done diff --git a/e2e-tests/tests/pitr-minio/98-drop-finalizer.yaml b/e2e-tests/tests/pitr-minio/98-drop-finalizer.yaml new file mode 100644 index 000000000..b235ac0b7 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/98-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: pitr-minio + finalizers: [] diff --git a/e2e-tests/tests/pitr-minio/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/pitr-minio/99-remove-cluster-gracefully.yaml new file mode 100644 index 000000000..3ebb90e80 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/99-remove-cluster-gracefully.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: ps.percona.com/v1 + kind: PerconaServerMySQL + metadata: + name: pitr-minio +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + destroy_operator + timeout: 60 diff --git a/pkg/binlogserver/search.go b/pkg/binlogserver/search.go index 5c70adbb1..a37cf2e0c 100644 --- a/pkg/binlogserver/search.go +++ b/pkg/binlogserver/search.go @@ -53,7 +53,7 @@ func execSearch(ctx context.Context, cl client.Client, cliCmd clientcmd.Client, var stdout, stderr bytes.Buffer if err := cliCmd.Exec(ctx, pod, AppName, cmd, nil, &stdout, &stderr, false); err != nil { - return nil, errors.Wrapf(err, "exec binlog_server %s: stderr: %s", subcommand, stderr.String()) + return nil, errors.Wrapf(err, "exec binlog_server %s: stdout: %s stderr: %s", subcommand, stdout.String(), stderr.String()) } var resp SearchResponse diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index 8f8571f04..f598f145d 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -180,9 +180,12 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req defer r.sm.Delete(cr.Spec.ClusterName) if cr.Spec.PITR != nil { + status.State = apiv1.RestoreStarting if err := r.reconcilePITRConfig(ctx, cr, cluster); err != nil { + status.StateDesc = errors.Wrap(err, "reconcile pitr config").Error() return ctrl.Result{}, errors.Wrap(err, "reconcile pitr config") } + status.StateDesc = "" } log.Info("Pausing cluster", "cluster", cluster.Name) From 7cc8a91ab73f2bfb2933ea136fadbe57e5dc51e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 23 Mar 2026 19:54:35 +0300 Subject: [PATCH 019/102] fix test issues --- .../tests/pitr-minio/01-deploy-operator.yaml | 1 - e2e-tests/tests/pitr-minio/02-assert.yaml | 16 ---------------- .../tests/pitr-minio/02-create-cluster.yaml | 5 ++--- .../tests/pitr-minio/06-create-pitr-restore.yaml | 4 ++-- 4 files changed, 4 insertions(+), 22 deletions(-) diff --git a/e2e-tests/tests/pitr-minio/01-deploy-operator.yaml b/e2e-tests/tests/pitr-minio/01-deploy-operator.yaml index c96e1f6b9..6ab1b37f9 100644 --- a/e2e-tests/tests/pitr-minio/01-deploy-operator.yaml +++ b/e2e-tests/tests/pitr-minio/01-deploy-operator.yaml @@ -10,7 +10,6 @@ commands: apply_s3_storage_secrets deploy_operator - deploy_tls_cluster_secrets deploy_client deploy_minio timeout: 300 diff --git a/e2e-tests/tests/pitr-minio/02-assert.yaml b/e2e-tests/tests/pitr-minio/02-assert.yaml index 9b7df6774..05e4df507 100644 --- a/e2e-tests/tests/pitr-minio/02-assert.yaml +++ b/e2e-tests/tests/pitr-minio/02-assert.yaml @@ -16,18 +16,6 @@ status: --- kind: StatefulSet apiVersion: apps/v1 -metadata: - name: pitr-minio-orc -status: - observedGeneration: 1 - replicas: 3 - readyReplicas: 3 - currentReplicas: 3 - updatedReplicas: 3 - collisionCount: 0 ---- -kind: StatefulSet -apiVersion: apps/v1 metadata: name: pitr-minio-haproxy status: @@ -65,8 +53,4 @@ status: ready: 3 size: 3 state: ready - orchestrator: - ready: 3 - size: 3 - state: ready state: ready diff --git a/e2e-tests/tests/pitr-minio/02-create-cluster.yaml b/e2e-tests/tests/pitr-minio/02-create-cluster.yaml index 384e81518..4ec9c500a 100644 --- a/e2e-tests/tests/pitr-minio/02-create-cluster.yaml +++ b/e2e-tests/tests/pitr-minio/02-create-cluster.yaml @@ -9,12 +9,10 @@ commands: source ../../functions get_cr \ - | yq eval '.spec.mysql.clusterType="async"' - \ + | yq eval '.spec.mysql.clusterType="group-replication"' - \ | yq eval ".spec.mysql.size=3" - \ | yq eval ".spec.proxy.haproxy.enabled=true" - \ | yq eval ".spec.proxy.haproxy.size=3" - \ - | yq eval ".spec.orchestrator.enabled=true" - \ - | yq eval ".spec.orchestrator.size=3" - \ | yq eval ".spec.backup.backoffLimit=3" - \ | yq eval '.spec.backup.storages.minio.type="s3"' - \ | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ @@ -25,6 +23,7 @@ commands: | yq eval '.spec.backup.storages.minio.containerOptions.env[0].value="false"' - \ | yq eval '.spec.backup.pitr.enabled=true' - \ | yq eval '.spec.backup.pitr.binlogServer.storage.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.prefix="binlogs"' - \ | yq eval '.spec.backup.pitr.binlogServer.storage.s3.credentialsSecret="minio-secret"' - \ | yq eval ".spec.backup.pitr.binlogServer.storage.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ | yq eval '.spec.backup.pitr.binlogServer.storage.s3.region="us-east-1"' - \ diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml index 2d0b7c63f..4343544af 100644 --- a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml @@ -1,8 +1,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 180 commands: - - script: |- + - timeout: 90 + script: |- set -o errexit set -o xtrace From e5d30c0650ffc3fd3605df85901a2f07186fd6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 23 Mar 2026 20:00:58 +0300 Subject: [PATCH 020/102] fix one more issue --- e2e-tests/tests/pitr-minio/06-assert.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/e2e-tests/tests/pitr-minio/06-assert.yaml b/e2e-tests/tests/pitr-minio/06-assert.yaml index 0c371a120..8f67aa564 100644 --- a/e2e-tests/tests/pitr-minio/06-assert.yaml +++ b/e2e-tests/tests/pitr-minio/06-assert.yaml @@ -17,10 +17,6 @@ status: ready: 3 size: 3 state: ready - orchestrator: - ready: 3 - size: 3 - state: ready state: ready --- kind: PerconaServerMySQLRestore From 7bd1bcf754c77700320ffbb8d2e4df442e39ae5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 24 Mar 2026 15:58:58 +0300 Subject: [PATCH 021/102] fix type=date --- cmd/pitr/main.go | 55 +++++++++++--------------- pkg/controller/psrestore/controller.go | 4 +- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 10f6af26c..65bf54d6f 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -9,8 +9,6 @@ import ( "net/url" "os" "os/exec" - "regexp" - "strconv" "strings" "time" @@ -162,13 +160,12 @@ func runApply(ctx context.Context) error { lastRelayLog := fmt.Sprintf("%s-relay-bin.%06d", hostname, len(entries)) lastRelayLogPath := fmt.Sprintf("/var/lib/mysql/%s", lastRelayLog) - var stopPos int if pitrType == "date" { - stopPos, err = getStopPosition(lastRelayLogPath, pitrDate) + pitrGTID, err = getLatestGTIDByDatetime(lastRelayLogPath, pitrDate) if err != nil { - return fmt.Errorf("get stop position: %w", err) + return fmt.Errorf("get latest GTID for date %s: %w", pitrDate, err) } - log.Printf("stop position for date %s: %s %d", pitrDate, lastRelayLog, stopPos) + log.Printf("latest GTID for date %s: %s", pitrDate, pitrGTID) } firstRelayLog := fmt.Sprintf("%s-relay-bin.000001", hostname) @@ -178,19 +175,9 @@ func runApply(ctx context.Context) error { return fmt.Errorf("change replication source: %w", err) } - switch pitrType { - case "gtid": - log.Printf("starting replica until GTID: %s", pitrGTID) - if err := database.StartReplicaUntilGTID(ctx, pitrGTID); err != nil { - return fmt.Errorf("start replica until GTID: %w", err) - } - case "date": - log.Printf("starting replica until position: %s:%d", lastRelayLog, stopPos) - if err := database.StartReplicaUntilPosition(ctx, lastRelayLog, stopPos); err != nil { - return fmt.Errorf("start replica until position: %w", err) - } - default: - return fmt.Errorf("unsupported PITR type: %s", pitrType) + log.Printf("starting replica until GTID: %s", pitrGTID) + if err := database.StartReplicaUntilGTID(ctx, pitrGTID); err != nil { + return fmt.Errorf("start replica until GTID: %w", err) } log.Println("waiting for replication to complete...") @@ -212,26 +199,30 @@ func runApply(ctx context.Context) error { return nil } -func getStopPosition(relayLogPath, stopDatetime string) (int, error) { - cmd := exec.Command("mysqlbinlog", "--stop-datetime="+stopDatetime, relayLogPath) - out, err := cmd.Output() +func getLatestGTIDByDatetime(relayLogPath, stopDatetime string) (string, error) { + cmd := exec.Command("bash", "-c", + fmt.Sprintf("mysqlbinlog --stop-datetime='%s' %s | grep GTID_NEXT | grep -v AUTOMATIC | tail -n 1", + stopDatetime, relayLogPath)) + + output, err := cmd.Output() if err != nil { - return 0, fmt.Errorf("run mysqlbinlog: %w", err) + return "", fmt.Errorf("failed to execute mysqlbinlog pipeline: %w", err) } - re := regexp.MustCompile(`end_log_pos (\d+)`) - matches := re.FindAllStringSubmatch(string(out), -1) - if len(matches) == 0 { - return 0, fmt.Errorf("no end_log_pos found in mysqlbinlog output") + line := strings.TrimSpace(string(output)) + if line == "" { + return "", fmt.Errorf("no GTID found before %s in %s", stopDatetime, relayLogPath) } - lastMatch := matches[len(matches)-1] - pos, err := strconv.Atoi(lastMatch[1]) - if err != nil { - return 0, fmt.Errorf("parse end_log_pos: %w", err) + // Extract GTID from: SET @@SESSION.GTID_NEXT= 'uuid:n,uuid:n'/*!*/; + start := strings.Index(line, "'") + end := strings.LastIndex(line, "'") + if start == -1 || end == -1 || start == end { + return "", fmt.Errorf("failed to parse GTID from line: %s", line) } - return pos, nil + gtid := line[start+1 : end] + return gtid, nil } // objectKeyFromURI extracts the S3 object key from a full URI. diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index f598f145d..9d50c1e7a 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "sync" "time" @@ -388,7 +389,8 @@ func (r *PerconaServerMySQLRestoreReconciler) searchBinlogs( switch cr.Spec.PITR.Type { case apiv1.PITRDate: - resp, err = binlogserver.SearchByTimestamp(ctx, r.Client, r.ClientCmd, cluster, cr.Spec.PITR.Date) + ts := strings.Replace(cr.Spec.PITR.Date, " ", "T", 1) + resp, err = binlogserver.SearchByTimestamp(ctx, r.Client, r.ClientCmd, cluster, ts) case apiv1.PITRGtid: resp, err = binlogserver.SearchByGTID(ctx, r.Client, r.ClientCmd, cluster, cr.Spec.PITR.GTID) default: From 30aa87c0864c4171e7f5afa4a08f63f913c59a5f Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 24 Mar 2026 17:17:13 +0200 Subject: [PATCH 022/102] strip the leading bucket_name to get the object key --- cmd/pitr/main.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 65bf54d6f..e4e67bc61 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -83,7 +83,7 @@ func runSetup(ctx context.Context) error { relayLogName := fmt.Sprintf("%s-relay-bin.%06d", hostname, i+1) relayLogPath := fmt.Sprintf("/var/lib/mysql/%s", relayLogName) - objectKey, err := objectKeyFromURI(entry.URI) + objectKey, err := objectKeyFromURI(entry.URI, bucket) if err != nil { return fmt.Errorf("parse URI %s: %w", entry.URI, err) } @@ -226,12 +226,13 @@ func getLatestGTIDByDatetime(relayLogPath, stopDatetime string) (string, error) } // objectKeyFromURI extracts the S3 object key from a full URI. -// e.g. "https://minio-service:9000/binlogs/binlog.000001" -> "binlogs/binlog.000001" -func objectKeyFromURI(uri string) (string, error) { +// e.g. "https://minio-service:9000/bucket/binlogs/binlog.000001" -> "binlogs/binlog.000001" +func objectKeyFromURI(uri, bucket string) (string, error) { u, err := url.Parse(uri) if err != nil { return "", fmt.Errorf("parse URL: %w", err) } - // Path starts with "/", trim the leading slash to get the object key - return strings.TrimPrefix(u.Path, "/"), nil + // Path is "//", strip the leading "//" to get the object key + key := strings.TrimPrefix(u.Path, "/"+bucket+"/") + return key, nil } From f97ad4ed425662c294ebfd5b4afe3cb398cfa976 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 24 Mar 2026 19:28:02 +0200 Subject: [PATCH 023/102] fix e2e test --- e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml | 2 -- e2e-tests/tests/pitr-minio/07-assert.yaml | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml index 4343544af..5ba8f6596 100644 --- a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml @@ -10,8 +10,6 @@ commands: cluster_name=$(get_cluster_name) - PITR_GTID=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_haproxy_svc ${cluster_name})" | tr -d '\n') - run_mysql \ "INSERT myDB.myTable (id) VALUES (100502)" \ "-h $(get_haproxy_svc ${cluster_name})" diff --git a/e2e-tests/tests/pitr-minio/07-assert.yaml b/e2e-tests/tests/pitr-minio/07-assert.yaml index bbd6bf1cc..75528883a 100644 --- a/e2e-tests/tests/pitr-minio/07-assert.yaml +++ b/e2e-tests/tests/pitr-minio/07-assert.yaml @@ -7,18 +7,18 @@ apiVersion: v1 metadata: name: 07-read-data-0 data: - max_id: "100501" + max_id: "100500" --- kind: ConfigMap apiVersion: v1 metadata: name: 07-read-data-1 data: - max_id: "100501" + max_id: "100500" --- kind: ConfigMap apiVersion: v1 metadata: name: 07-read-data-2 data: - max_id: "100501" + max_id: "100500" From 29df4dc4e75fcf95c873e018a1817e1f7bd7a114 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 24 Mar 2026 20:34:16 +0200 Subject: [PATCH 024/102] add back PITR_GTID --- e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml index 5ba8f6596..fbf2bcafb 100644 --- a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml @@ -14,6 +14,8 @@ commands: "INSERT myDB.myTable (id) VALUES (100502)" \ "-h $(get_haproxy_svc ${cluster_name})" + PITR_GTID=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_haproxy_svc ${cluster_name})" | tr -d '\n') + sleep 60 echo '{}' \ From 78338f734954c9f66e36b68733590b56b6cb3cfe Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 24 Mar 2026 20:41:24 +0200 Subject: [PATCH 025/102] update e2e tests list and ordering --- e2e-tests/run-distro.csv | 5 +++-- e2e-tests/run-release.csv | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/e2e-tests/run-distro.csv b/e2e-tests/run-distro.csv index fa835947b..440325046 100644 --- a/e2e-tests/run-distro.csv +++ b/e2e-tests/run-distro.csv @@ -1,7 +1,9 @@ +async-upgrade auto-config config config-router demand-backup-minio +pitr-minio demand-backup-cloud demand-backup-retry gr-demand-backup-minio @@ -19,7 +21,6 @@ gr-self-healing gr-tls-cert-manager gr-users gr-upgrade -async-upgrade haproxy init-deploy one-pod @@ -32,4 +33,4 @@ sidecars smart-update storage tls-cert-manager -users +users \ No newline at end of file diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index eed46ec42..e899192c3 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -1,10 +1,11 @@ -version-service async-ignore-annotations async-global-metadata +async-upgrade auto-config config config-router demand-backup-minio +pitr-minio demand-backup-cloud demand-backup-retry async-data-at-rest-encryption @@ -26,13 +27,13 @@ gr-self-healing gr-tls-cert-manager gr-users gr-upgrade -async-upgrade haproxy init-deploy limits monitoring one-pod operator-self-healing +pvc-resize recreate scaling scheduled-backup @@ -43,4 +44,4 @@ storage telemetry tls-cert-manager users -pvc-resize +version-service \ No newline at end of file From a001e397c012f837765d858ad5f31f526bbdd9c0 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 24 Mar 2026 22:02:00 +0200 Subject: [PATCH 026/102] fixes and unit test fixes --- pkg/binlogserver/search.go | 6 +++--- pkg/controller/ps/controller.go | 27 ++++++++++++++++----------- pkg/controller/ps/controller_test.go | 10 +++++++++- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/pkg/binlogserver/search.go b/pkg/binlogserver/search.go index a37cf2e0c..2db2e1fc6 100644 --- a/pkg/binlogserver/search.go +++ b/pkg/binlogserver/search.go @@ -19,9 +19,9 @@ import ( const binlogServerBinary = "/usr/bin/binlog_server" type SearchResponse struct { - Version int `json:"version"` - Status string `json:"status"` - Result []BinlogEntry `json:"result"` + Version int `json:"version"` + Status string `json:"status"` + Result []BinlogEntry `json:"result"` } type BinlogEntry struct { diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index c8461c6e8..f63725a1d 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1290,16 +1290,6 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context logger := logf.FromContext(ctx) - if cr.Status.MySQL.Ready < 1 { - logger.V(1).Info("Waiting for at least one MySQL pod to be ready") - return nil - } - - if len(cr.Status.Host) == 0 { - logger.V(1).Info("Waiting for .status.host to be populated") - return nil - } - s3 := cr.Spec.Backup.PiTR.BinlogServer.Storage.S3 if s3 == nil || len(s3.CredentialsSecret) == 0 { @@ -1394,6 +1384,16 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context return errors.Wrap(err, "reconcile secret") } + if cr.Status.MySQL.Ready < 1 { + logger.V(1).Info("Waiting for at least one MySQL pod to be ready") + return nil + } + + if len(cr.Status.Host) == 0 { + logger.V(1).Info("Waiting for .status.host to be populated") + return nil + } + initImage, err := k8s.InitImage(ctx, r.Client, cr, &cr.Spec.Backup.PiTR.BinlogServer.PodSpec) if err != nil { return errors.Wrap(err, "get init image") @@ -1422,7 +1422,12 @@ func (r *PerconaServerMySQLReconciler) cleanupBinlogServer(ctx context.Context, return nil } - if err := r.Delete(ctx, binlogserver.StatefulSet(cr, "", "")); err != nil && !k8serrors.IsNotFound(err) { + if err := r.Delete(ctx, &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: binlogserver.Name(cr), + Namespace: cr.Namespace, + }, + }); err != nil && !k8serrors.IsNotFound(err) { return errors.Wrap(err, "failed to delete binlog server statefulset") } diff --git a/pkg/controller/ps/controller_test.go b/pkg/controller/ps/controller_test.go index 7ebe70c17..6e0dc2106 100644 --- a/pkg/controller/ps/controller_test.go +++ b/pkg/controller/ps/controller_test.go @@ -989,7 +989,7 @@ var _ = Describe("Reconcile Binlog Server", Ordered, func() { S3: &psv1.BackupStorageS3Spec{ Bucket: "s3-test-bucket", Region: "us-west-1", - EndpointURL: "s3.amazonaws.com", + EndpointURL: "https://s3.amazonaws.com", CredentialsSecret: "s3-test-credentials", }, }, @@ -1021,6 +1021,14 @@ var _ = Describe("Reconcile Binlog Server", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) + It("should set MySQL status as ready", func() { + fetchedCR := cr.DeepCopy() + Expect(k8sClient.Get(ctx, crNamespacedName, fetchedCR)).Should(Succeed()) + fetchedCR.Status.MySQL.Ready = 1 + fetchedCR.Status.Host = mysql.FQDN(fetchedCR, 0) + Expect(k8sClient.Status().Update(ctx, fetchedCR)).Should(Succeed()) + }) + It("should create secret for Binlog Server configuration", func() { _, err = reconciler().Reconcile(ctx, ctrl.Request{NamespacedName: crNamespacedName}) Expect(err).NotTo(HaveOccurred()) From 43ceaf7871bab44f853205f0ffaf4cc395127f86 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 24 Mar 2026 22:12:06 +0200 Subject: [PATCH 027/102] fix linter --- cmd/internal/db/db.go | 6 +++++- cmd/pitr/main.go | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/cmd/internal/db/db.go b/cmd/internal/db/db.go index b156ada3c..429045383 100644 --- a/cmd/internal/db/db.go +++ b/cmd/internal/db/db.go @@ -431,7 +431,11 @@ func (d *DB) WaitReplicaSQLThreadStop(ctx context.Context, pollInterval time.Dur if err != nil { return errors.Wrap(err, "query replication applier worker status") } - defer rows.Close() + defer func() { + if err := rows.Close(); err != nil { + logf.FromContext(ctx).Error(err, "close rows") + } + }() for rows.Next() { var errNum int diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index e4e67bc61..3142eb0a8 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -97,13 +97,19 @@ func runSetup(ctx context.Context) error { f, err := os.Create(relayLogPath) if err != nil { - obj.Close() + if closeErr := obj.Close(); closeErr != nil { + log.Printf("close object %s: %v", entry.URI, closeErr) + } return fmt.Errorf("create relay log file %s: %w", relayLogPath, err) } _, err = io.Copy(f, obj) - obj.Close() - f.Close() + if closeErr := obj.Close(); closeErr != nil { + log.Printf("close object %s: %v", entry.URI, closeErr) + } + if closeErr := f.Close(); closeErr != nil { + log.Printf("close relay log file %s: %v", relayLogPath, closeErr) + } if err != nil { return fmt.Errorf("write relay log file %s: %w", relayLogPath, err) } @@ -144,7 +150,11 @@ func runApply(ctx context.Context) error { if err != nil { return fmt.Errorf("connect to MySQL: %w", err) } - defer database.Close() + defer func() { + if err := database.Close(); err != nil { + log.Printf("close database: %v", err) + } + }() binlogsPath := os.Getenv("BINLOGS_PATH") data, err := os.ReadFile(binlogsPath) From b15ce3c042063c75bf554ec0fbf6295400ee7951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Wed, 25 Mar 2026 16:09:47 +0300 Subject: [PATCH 028/102] add IMAGE_BINLOG_SERVER --- e2e-tests/functions | 5 ++++- e2e-tests/vars.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e-tests/functions b/e2e-tests/functions index de9b0ed8c..c8e3202b1 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -637,7 +637,8 @@ get_cr() { local image_toolkit=${6:-${IMAGE_TOOLKIT}} local image_haproxy=${7:-${IMAGE_HAPROXY}} local image_pmm_client=${8:-${IMAGE_PMM_CLIENT}} - local cr_file=${9:-${DEPLOY_DIR}/cr.yaml} + local image_binlog_server=${9:-${IMAGE_BINLOG_SERVER}} + local cr_file=${10:-${DEPLOY_DIR}/cr.yaml} yq eval "$(printf '.metadata.name="%s"' "${test_name}${name_suffix:+-$name_suffix}")" ${cr_file} \ | yq eval "$(printf '.spec.initContainer.image="%s"' "${IMAGE}")" - \ @@ -654,6 +655,7 @@ get_cr() { | yq eval "$(printf '.spec.toolkit.image="%s"' "${image_toolkit}")" - \ | yq eval "$(printf '.spec.proxy.haproxy.image="%s"' "${image_haproxy}")" - \ | yq eval "$(printf '.spec.pmm.image="%s"' "${image_pmm_client}")" - \ + | yq eval "$(printf '.spec.backup.pitr.binlogServer.image="%s"' "${image_binlog_server}")" - \ | if [ -n "${MINIKUBE}" ]; then yq eval '(.. | select(has("antiAffinityTopologyKey")).antiAffinityTopologyKey) |= "none"' - \ | yq eval '.spec.proxy.haproxy.resources.requests.cpu="300m"' - @@ -1553,6 +1555,7 @@ get_cr_with_latest_versions_in_vs() { ${image_toolkit} \ ${image_haproxy} \ ${image_pmm_client} \ + ${image_binlog_server} \ ${TEMP_DIR}/cr.yaml } diff --git a/e2e-tests/vars.sh b/e2e-tests/vars.sh index bcbc5d7e4..6577d98e0 100755 --- a/e2e-tests/vars.sh +++ b/e2e-tests/vars.sh @@ -22,6 +22,7 @@ export IMAGE_ORCHESTRATOR=${IMAGE_ORCHESTRATOR:-"perconalab/percona-server-mysql export IMAGE_ROUTER=${IMAGE_ROUTER:-"perconalab/percona-server-mysql-operator:main-router${MYSQL_VERSION}"} export IMAGE_TOOLKIT=${IMAGE_TOOLKIT:-"perconalab/percona-server-mysql-operator:main-toolkit"} export IMAGE_HAPROXY=${IMAGE_HAPROXY:-"perconalab/percona-server-mysql-operator:main-haproxy"} +export IMAGE_BINLOG_SERVER=${IMAGE_BINLOG_SERVER:-"perconalab/percona-binlog-server:0.2.0"} export PMM_SERVER_VERSION=${PMM_SERVER_VERSION:-"1.4.3"} export IMAGE_PMM_CLIENT=${IMAGE_PMM_CLIENT:-"perconalab/pmm-client:3-dev-latest"} export IMAGE_PMM_SERVER=${IMAGE_PMM_SERVER:-"perconalab/pmm-server:3-dev-latest"} From 4c2315e74b2da1f821e553f48982f8bb80e58d9e Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 26 Mar 2026 10:43:10 +0200 Subject: [PATCH 029/102] update e2e test to assert date pitr --- .../pitr-minio/06-create-pitr-restore.yaml | 4 +++ e2e-tests/tests/pitr-minio/08-assert.yaml | 27 +++++++++++++++++++ .../pitr-minio/08-create-date-restore.yaml | 23 ++++++++++++++++ e2e-tests/tests/pitr-minio/09-assert.yaml | 24 +++++++++++++++++ e2e-tests/tests/pitr-minio/09-read-data.yaml | 16 +++++++++++ 5 files changed, 94 insertions(+) create mode 100644 e2e-tests/tests/pitr-minio/08-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/08-create-date-restore.yaml create mode 100644 e2e-tests/tests/pitr-minio/09-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/09-read-data.yaml diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml index fbf2bcafb..52bab6b0e 100644 --- a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml @@ -16,6 +16,10 @@ commands: PITR_GTID=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_haproxy_svc ${cluster_name})" | tr -d '\n') + sleep 1 + PITR_DATE=$(date -u '+%Y-%m-%d %H:%M:%S') + kubectl create configmap -n "${NAMESPACE}" pitr-date --from-literal=date="${PITR_DATE}" + sleep 60 echo '{}' \ diff --git a/e2e-tests/tests/pitr-minio/08-assert.yaml b/e2e-tests/tests/pitr-minio/08-assert.yaml new file mode 100644 index 000000000..e48c39b08 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/08-assert.yaml @@ -0,0 +1,27 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + state: ready +--- +kind: PerconaServerMySQLRestore +apiVersion: ps.percona.com/v1 +metadata: + name: pitr-minio-restore-date +status: + state: Succeeded diff --git a/e2e-tests/tests/pitr-minio/08-create-date-restore.yaml b/e2e-tests/tests/pitr-minio/08-create-date-restore.yaml new file mode 100644 index 000000000..601ca5f53 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/08-create-date-restore.yaml @@ -0,0 +1,23 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 90 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + PITR_DATE=$(kubectl get configmap -n "${NAMESPACE}" pitr-date -o jsonpath='{.data.date}') + + echo '{}' \ + | yq eval '.apiVersion = "ps.percona.com/v1"' - \ + | yq eval '.kind = "PerconaServerMySQLRestore"' - \ + | yq eval '.metadata.name = "pitr-minio-restore-date"' - \ + | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ + | yq eval '.spec.backupName = "pitr-minio-backup"' - \ + | yq eval '.spec.pitr.type = "date"' - \ + | yq eval ".spec.pitr.date = \"${PITR_DATE}\"" - \ + | kubectl apply -n "${NAMESPACE}" -f - \ No newline at end of file diff --git a/e2e-tests/tests/pitr-minio/09-assert.yaml b/e2e-tests/tests/pitr-minio/09-assert.yaml new file mode 100644 index 000000000..58b18699b --- /dev/null +++ b/e2e-tests/tests/pitr-minio/09-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 09-read-data-0 +data: + max_id: "100501" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 09-read-data-1 +data: + max_id: "100501" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 09-read-data-2 +data: + max_id: "100501" diff --git a/e2e-tests/tests/pitr-minio/09-read-data.yaml b/e2e-tests/tests/pitr-minio/09-read-data.yaml new file mode 100644 index 000000000..8d79b4d3f --- /dev/null +++ b/e2e-tests/tests/pitr-minio/09-read-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + for i in 0 1 2; do + max_id=$(run_mysql "SELECT MAX(id) FROM myDB.myTable" "-h ${cluster_name}-mysql-${i}.${cluster_name}-mysql") + kubectl create configmap -n "${NAMESPACE}" 09-read-data-${i} --from-literal=max_id="${max_id}" + done From 9658e72c2c6eecb6a00b041f1d09094684650cac Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 26 Mar 2026 11:06:00 +0200 Subject: [PATCH 030/102] add connection of pbs to primary service + unit test --- pkg/controller/ps/controller.go | 2 +- pkg/controller/ps/controller_test.go | 121 +++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index 12dbc15f0..bb708fe0b 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1341,7 +1341,7 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context File: "/dev/stdout", }, Connection: binlogserver.Connection{ - Host: mysql.FQDN(cr, 0), + Host: fmt.Sprintf("%s.%s", mysql.PrimaryServiceName(cr), cr.Namespace), Port: 3306, User: string(apiv1.UserReplication), Password: replPass, diff --git a/pkg/controller/ps/controller_test.go b/pkg/controller/ps/controller_test.go index 6e0dc2106..12cb89a84 100644 --- a/pkg/controller/ps/controller_test.go +++ b/pkg/controller/ps/controller_test.go @@ -18,6 +18,7 @@ package ps import ( "context" + "encoding/json" "fmt" "strconv" "strings" @@ -26,6 +27,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" gs "github.com/onsi/gomega/gstruct" + "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" + "github.com/percona/percona-server-mysql-operator/pkg/secret" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" @@ -1934,3 +1937,121 @@ var _ = Describe("Global labels and annotations", Ordered, func() { }) }) }) + +var _ = Describe("BinlogServer", Ordered, func() { + ctx := context.Background() + + const crName = "pitr-test" + const ns = crName + crNamespacedName := types.NamespacedName{Name: crName, Namespace: ns} + + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + } + + BeforeAll(func() { + By("Creating the Namespace") + Expect(k8sClient.Create(ctx, namespace)).To(Succeed()) + }) + + AfterAll(func() { + _ = k8sClient.Delete(ctx, namespace) + }) + + cr, err := readDefaultCR(crName, ns) + It("should read default cr.yaml", func() { + Expect(err).NotTo(HaveOccurred()) + }) + + It("should configure PiTR and create the CR", func() { + cr.Spec.Backup.PiTR.Enabled = true + cr.Spec.Backup.PiTR.BinlogServer = &psv1.BinlogServerSpec{ + Storage: psv1.BinlogServerStorageSpec{ + S3: &psv1.BackupStorageS3Spec{ + Bucket: "test-bucket", + Region: "us-east-1", + EndpointURL: "s3://s3.amazonaws.com", + CredentialsSecret: "s3-secret", + }, + }, + PodSpec: psv1.PodSpec{ + Size: 1, + ContainerSpec: psv1.ContainerSpec{ + Image: "perconalab/percona-binlog-server:0.2.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cr)).To(Succeed()) + }) + + It("should create the S3 credentials secret", func() { + s3Secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s3-secret", + Namespace: ns, + }, + Data: map[string][]byte{ + secret.CredentialsAWSAccessKey: []byte("access-key"), + secret.CredentialsAWSSecretKey: []byte("secret-key"), + }, + } + Expect(k8sClient.Create(ctx, s3Secret)).To(Succeed()) + }) + + It("should create the internal secret with the replication user password", func() { + internalSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.InternalSecretName(), + Namespace: ns, + }, + Data: map[string][]byte{ + string(psv1.UserReplication): []byte("repl-password"), + }, + } + Expect(k8sClient.Create(ctx, internalSecret)).To(Succeed()) + }) + + It("should set the binlog server connection host to the primary service", func() { + Expect(k8sClient.Get(ctx, crNamespacedName, cr)).To(Succeed()) + + Expect(reconciler().reconcileBinlogServer(ctx, cr)).To(Succeed()) + + configSecret := &corev1.Secret{} + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: binlogserver.ConfigSecretName(cr), + Namespace: ns, + }, configSecret) + return err == nil + }, time.Second*15, time.Millisecond*250).Should(BeTrue()) + + var config binlogserver.Configuration + Expect(json.Unmarshal(configSecret.Data[binlogserver.ConfigKey], &config)).To(Succeed()) + + Expect(config.Connection.Host).To(Equal(fmt.Sprintf("%s.%s", mysql.PrimaryServiceName(cr), ns))) + }) + + It("should create the binlog server StatefulSet once MySQL is ready", func() { + Expect(k8sClient.Get(ctx, crNamespacedName, cr)).To(Succeed()) + + cr.Status.MySQL.Ready = 1 + cr.Status.Host = "pitr-test-haproxy.pitr-test" + Expect(k8sClient.Status().Update(ctx, cr)).To(Succeed()) + + Expect(k8sClient.Get(ctx, crNamespacedName, cr)).To(Succeed()) + Expect(reconciler().reconcileBinlogServer(ctx, cr)).To(Succeed()) + + sts := &appsv1.StatefulSet{} + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: binlogserver.Name(cr), + Namespace: ns, + }, sts) + return err == nil + }, time.Second*15, time.Millisecond*250).Should(BeTrue()) + + Expect(sts.Spec.Replicas).To(gs.PointTo(BeEquivalentTo(1))) + }) +}) From 692053cbf5b49a5a8b39bc8f5dde1fd86387cb04 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 26 Mar 2026 16:26:16 +0100 Subject: [PATCH 031/102] add sans cert for primary service --- pkg/controller/ps/tls_test.go | 6 +++ pkg/tls/tls.go | 3 ++ pkg/tls/tls_test.go | 78 +++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 pkg/tls/tls_test.go diff --git a/pkg/controller/ps/tls_test.go b/pkg/controller/ps/tls_test.go index 6450fbf99..6a86c129a 100644 --- a/pkg/controller/ps/tls_test.go +++ b/pkg/controller/ps/tls_test.go @@ -78,6 +78,9 @@ var _ = Describe("TLS secrets without cert-manager", Ordered, func() { "*.ps-cluster1-mysql", "*.ps-cluster1-mysql.tls-1", "*.ps-cluster1-mysql.tls-1.svc", + "ps-cluster1-mysql-primary", + "ps-cluster1-mysql-primary.tls-1", + "ps-cluster1-mysql-primary.tls-1.svc", "*.ps-cluster1-orchestrator", "*.ps-cluster1-orchestrator.tls-1", "*.ps-cluster1-orchestrator.tls-1.svc", @@ -126,6 +129,9 @@ var _ = Describe("TLS secrets without cert-manager", Ordered, func() { "*.ps-cluster1-mysql", "*.ps-cluster1-mysql.tls-1", "*.ps-cluster1-mysql.tls-1.svc", + "ps-cluster1-mysql-primary", + "ps-cluster1-mysql-primary.tls-1", + "ps-cluster1-mysql-primary.tls-1.svc", "*.ps-cluster1-orchestrator", "*.ps-cluster1-orchestrator.tls-1", "*.ps-cluster1-orchestrator.tls-1.svc", diff --git a/pkg/tls/tls.go b/pkg/tls/tls.go index aa4d63d99..78d4b266c 100644 --- a/pkg/tls/tls.go +++ b/pkg/tls/tls.go @@ -29,6 +29,9 @@ func DNSNames(cr *apiv1.PerconaServerMySQL) []string { fmt.Sprintf("*.%s-mysql", cr.Name), fmt.Sprintf("*.%s-mysql.%s", cr.Name, cr.Namespace), fmt.Sprintf("*.%s-mysql.%s.svc", cr.Name, cr.Namespace), + fmt.Sprintf("%s-mysql-primary", cr.Name), + fmt.Sprintf("%s-mysql-primary.%s", cr.Name, cr.Namespace), + fmt.Sprintf("%s-mysql-primary.%s.svc", cr.Name, cr.Namespace), fmt.Sprintf("*.%s-orchestrator", cr.Name), fmt.Sprintf("*.%s-orchestrator.%s", cr.Name, cr.Namespace), fmt.Sprintf("*.%s-orchestrator.%s.svc", cr.Name, cr.Namespace), diff --git a/pkg/tls/tls_test.go b/pkg/tls/tls_test.go new file mode 100644 index 000000000..090843a96 --- /dev/null +++ b/pkg/tls/tls_test.go @@ -0,0 +1,78 @@ +package tls + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" +) + +func TestDNSNames(t *testing.T) { + tests := map[string]struct { + cr *apiv1.PerconaServerMySQL + expected map[string]struct{} + }{ + "no extra SANs": { + cr: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + expected: map[string]struct{}{ + "*.cluster1-mysql": {}, + "*.cluster1-mysql.default": {}, + "*.cluster1-mysql.default.svc": {}, + "cluster1-mysql-primary": {}, + "cluster1-mysql-primary.default": {}, + "cluster1-mysql-primary.default.svc": {}, + "*.cluster1-orchestrator": {}, + "*.cluster1-orchestrator.default": {}, + "*.cluster1-orchestrator.default.svc": {}, + "*.cluster1-router": {}, + "*.cluster1-router.default": {}, + "*.cluster1-router.default.svc": {}, + }, + }, + "with extra SANs": { + cr: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: apiv1.PerconaServerMySQLSpec{ + TLS: &apiv1.TLSSpec{ + SANs: []string{"extra.example.com"}, + }, + }, + }, + expected: map[string]struct{}{ + "*.cluster1-mysql": {}, + "*.cluster1-mysql.default": {}, + "*.cluster1-mysql.default.svc": {}, + "cluster1-mysql-primary": {}, + "cluster1-mysql-primary.default": {}, + "cluster1-mysql-primary.default.svc": {}, + "*.cluster1-orchestrator": {}, + "*.cluster1-orchestrator.default": {}, + "*.cluster1-orchestrator.default.svc": {}, + "*.cluster1-router": {}, + "*.cluster1-router.default": {}, + "*.cluster1-router.default.svc": {}, + "extra.example.com": {}, + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + actual := make(map[string]struct{}) + for _, n := range DNSNames(tt.cr) { + actual[n] = struct{}{} + } + assert.Equal(t, tt.expected, actual) + }) + } +} From 9e018989aea34a39ee2a43bd52fd245bd1f4d3f3 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 26 Mar 2026 17:25:52 +0100 Subject: [PATCH 032/102] default s3:// when endpoint url is not configured --- pkg/controller/ps/controller.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index bb708fe0b..8d6427f4d 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1310,14 +1310,9 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context accessKey := s3Secret.Data[secret.CredentialsAWSAccessKey] secretKey := s3Secret.Data[secret.CredentialsAWSSecretKey] - protocol, host, err := parseEndpointURL(s3.EndpointURL) + s3Uri, err := s3URI(*s3, accessKey, secretKey) if err != nil { - return errors.Wrap(err, "parse endpoint URL") - } - - s3Uri := fmt.Sprintf("%s://%s:%s@%s/%s", protocol, accessKey, secretKey, host, s3.Bucket) - if len(s3.Prefix) > 0 { - s3Uri += fmt.Sprintf("/%s", s3.Prefix) + return errors.Wrap(err, "get s3 uri") } replPass, err := k8s.UserPassword(ctx, r.Client, cr, apiv1.UserReplication) @@ -1697,3 +1692,23 @@ func getPodIndexFromHostname(hostname string) (int, error) { return idx, nil } + +func s3URI(s3 apiv1.BackupStorageS3Spec, accessKey, secretKey []byte) (string, error) { + bucket := string(s3.Bucket) + if len(s3.Region) > 0 { + bucket = fmt.Sprintf("%s.%s", s3.Bucket, s3.Region) + } + uri := fmt.Sprintf("s3://%s:%s@%s", accessKey, secretKey, bucket) + if len(s3.EndpointURL) != 0 { + protocol, host, err := parseEndpointURL(s3.EndpointURL) + if err != nil { + return "", errors.Wrap(err, "parse endpoint URL") + } + uri = fmt.Sprintf("%s://%s:%s@%s/%s", protocol, accessKey, secretKey, host, s3.Bucket) + } + if len(s3.Prefix) > 0 { + uri += fmt.Sprintf("/%s", s3.Prefix) + } + + return uri, nil +} From d806f3d0031d2891701bf54c69018dc79b6c2102 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 26 Mar 2026 17:36:41 +0100 Subject: [PATCH 033/102] remove not used input from JobName --- pkg/controller/psrestore/controller.go | 2 +- pkg/pitr/pitr.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index 9d50c1e7a..ada42340c 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -324,7 +324,7 @@ func (r *PerconaServerMySQLRestoreReconciler) reconcilePITRJob( log := logf.FromContext(ctx) pitrJob := &batchv1.Job{} - nn := types.NamespacedName{Name: pitr.JobName(cluster, cr), Namespace: cr.Namespace} + nn := types.NamespacedName{Name: pitr.JobName(cr), Namespace: cr.Namespace} err := r.Get(ctx, nn, pitrJob) if err != nil { if !k8serrors.IsNotFound(err) { diff --git a/pkg/pitr/pitr.go b/pkg/pitr/pitr.go index 0f585a8a6..a7e2e0b7e 100644 --- a/pkg/pitr/pitr.go +++ b/pkg/pitr/pitr.go @@ -29,7 +29,7 @@ const ( BinlogsConfigKey = "binlogs.json" ) -func JobName(cluster *apiv1.PerconaServerMySQL, restore *apiv1.PerconaServerMySQLRestore) string { +func JobName(restore *apiv1.PerconaServerMySQLRestore) string { return fmt.Sprintf("pitr-restore-%s", restore.Name) } @@ -70,7 +70,7 @@ func RestoreJob( Kind: "Job", }, ObjectMeta: metav1.ObjectMeta{ - Name: JobName(cluster, restore), + Name: JobName(restore), Namespace: cluster.Namespace, Labels: labels, Annotations: util.SSMapMerge(cluster.GlobalAnnotations(), storage.Annotations), From fea7a94e9a7129e20d60a4b70a77b5008a4ae646 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 27 Mar 2026 11:56:46 +0100 Subject: [PATCH 034/102] encode aws creds and set ps status to init when binlog is not ready --- pkg/controller/ps/controller.go | 7 +- pkg/controller/ps/status.go | 14 +++ pkg/controller/ps/status_test.go | 184 +++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 2 deletions(-) diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index 8d6427f4d..52d92f23b 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -21,6 +21,7 @@ import ( "crypto/md5" "encoding/json" "fmt" + "net/url" "path" "slices" "strconv" @@ -1698,13 +1699,15 @@ func s3URI(s3 apiv1.BackupStorageS3Spec, accessKey, secretKey []byte) (string, e if len(s3.Region) > 0 { bucket = fmt.Sprintf("%s.%s", s3.Bucket, s3.Region) } - uri := fmt.Sprintf("s3://%s:%s@%s", accessKey, secretKey, bucket) + encodedAccessKey := url.QueryEscape(string(accessKey)) + encodedSecretKey := url.QueryEscape(string(secretKey)) + uri := fmt.Sprintf("s3://%s:%s@%s", encodedAccessKey, encodedSecretKey, bucket) if len(s3.EndpointURL) != 0 { protocol, host, err := parseEndpointURL(s3.EndpointURL) if err != nil { return "", errors.Wrap(err, "parse endpoint URL") } - uri = fmt.Sprintf("%s://%s:%s@%s/%s", protocol, accessKey, secretKey, host, s3.Bucket) + uri = fmt.Sprintf("%s://%s:%s@%s/%s", protocol, encodedAccessKey, encodedSecretKey, host, s3.Bucket) } if len(s3.Prefix) > 0 { uri += fmt.Sprintf("/%s", s3.Prefix) diff --git a/pkg/controller/ps/status.go b/pkg/controller/ps/status.go index 0e5a8bea6..028dd6beb 100644 --- a/pkg/controller/ps/status.go +++ b/pkg/controller/ps/status.go @@ -19,6 +19,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" database "github.com/percona/percona-server-mysql-operator/pkg/db" "github.com/percona/percona-server-mysql-operator/pkg/haproxy" "github.com/percona/percona-server-mysql-operator/pkg/innodbcluster" @@ -138,6 +139,15 @@ func (r *PerconaServerMySQLReconciler) reconcileCRStatus(ctx context.Context, cr } status.HAProxy = haproxyStatus + binlogServerStatus := apiv1.StatefulAppStatus{} + if cr.Spec.Backup.PiTR.Enabled { + binlogServerStatus, err = r.appStatus(ctx, cr, binlogserver.Name(cr), 1, binlogserver.MatchLabels(cr), status.BinlogServer.Version) + if err != nil { + return errors.Wrap(err, "get binlog server status") + } + } + status.BinlogServer = binlogServerStatus + status.State = apiv1.StateReady if cr.Spec.MySQL.IsAsync() { if cr.OrchestratorEnabled() && status.Orchestrator.State != apiv1.StateReady { @@ -155,6 +165,10 @@ func (r *PerconaServerMySQLReconciler) reconcileCRStatus(ctx context.Context, cr } } + if cr.Spec.Backup.PiTR.Enabled && status.BinlogServer.State != apiv1.StateReady { + status.State = apiv1.StateInitializing + } + if status.MySQL.State != apiv1.StateReady { status.State = status.MySQL.State } diff --git a/pkg/controller/ps/status_test.go b/pkg/controller/ps/status_test.go index 8c2b379cf..f73342fd6 100644 --- a/pkg/controller/ps/status_test.go +++ b/pkg/controller/ps/status_test.go @@ -15,6 +15,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -28,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" "github.com/percona/percona-server-mysql-operator/pkg/clientcmd" "github.com/percona/percona-server-mysql-operator/pkg/haproxy" "github.com/percona/percona-server-mysql-operator/pkg/innodbcluster" @@ -1255,6 +1257,9 @@ func makeFakeReadyPods(cr *apiv1.PerconaServerMySQL, amount int, podType string) case "router": pod.Name = router.PodName(cr, i) pod.Labels = router.Labels(cr) + case "binlogserver": + pod.Name = fmt.Sprintf("%s-%d", binlogserver.Name(cr), i) + pod.Labels = binlogserver.MatchLabels(cr) } pod.Namespace = cr.Namespace pods = append(pods, pod) @@ -1262,6 +1267,185 @@ func makeFakeReadyPods(cr *apiv1.PerconaServerMySQL, amount int, podType string) return pods } +func TestReconcileStatusBinlogServer(t *testing.T) { + ctx := context.Background() + + cr, err := readDefaultCR("ps-cluster1", "status-1") + require.NoError(t, err) + cr.Spec.MySQL.ClusterType = apiv1.ClusterTypeAsync + cr.Spec.UpdateStrategy = appsv1.OnDeleteStatefulSetStrategyType + cr.Spec.Backup.PiTR.Enabled = true + + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, apiv1.AddToScheme(scheme)) + + allReadyObjects := appendSlices( + makeFakeReadyPods(cr, 3, "mysql"), + makeFakeReadyPods(cr, 3, "haproxy"), + makeFakeReadyPods(cr, 3, "orchestrator"), + ) + + tests := map[string]struct { + cr *apiv1.PerconaServerMySQL + objects []client.Object + expected apiv1.PerconaServerMySQLStatus + }{ + "pitr enabled, binlog server pod not ready": { + cr: cr, + objects: allReadyObjects, + expected: apiv1.PerconaServerMySQLStatus{ + MySQL: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + Orchestrator: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + HAProxy: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + BinlogServer: apiv1.StatefulAppStatus{ + Size: 1, + State: apiv1.StateInitializing, + }, + State: apiv1.StateInitializing, + Host: cr.Name + "-haproxy." + cr.Namespace, + Conditions: []metav1.Condition{ + { + Type: apiv1.StateInitializing.String(), + Status: metav1.ConditionTrue, + Reason: apiv1.StateInitializing.String(), + }, + { + Type: apiv1.StateReady.String(), + Status: metav1.ConditionFalse, + Reason: apiv1.StateReady.String(), + }, + }, + }, + }, + "pitr enabled, binlog server pod ready": { + cr: cr, + objects: appendSlices( + allReadyObjects, + makeFakeReadyPods(cr, 1, "binlogserver"), + ), + expected: apiv1.PerconaServerMySQLStatus{ + MySQL: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + Orchestrator: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + HAProxy: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + BinlogServer: apiv1.StatefulAppStatus{ + Size: 1, + Ready: 1, + State: apiv1.StateReady, + }, + State: apiv1.StateReady, + Host: cr.Name + "-haproxy." + cr.Namespace, + Conditions: []metav1.Condition{ + { + Type: apiv1.StateInitializing.String(), + Status: metav1.ConditionFalse, + Reason: apiv1.StateInitializing.String(), + }, + { + Type: apiv1.StateReady.String(), + Status: metav1.ConditionTrue, + Reason: apiv1.StateReady.String(), + }, + }, + }, + }, + "pitr disabled, binlog server pod not ready does not affect cluster state": { + cr: updateResource(cr.DeepCopy(), func(cr *apiv1.PerconaServerMySQL) { + cr.Spec.Backup.PiTR.Enabled = false + }), + objects: allReadyObjects, + expected: apiv1.PerconaServerMySQLStatus{ + MySQL: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + Orchestrator: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + HAProxy: apiv1.StatefulAppStatus{ + Size: 3, + Ready: 3, + State: apiv1.StateReady, + }, + State: apiv1.StateReady, + Host: cr.Name + "-haproxy." + cr.Namespace, + Conditions: []metav1.Condition{ + { + Type: apiv1.StateInitializing.String(), + Status: metav1.ConditionFalse, + Reason: apiv1.StateInitializing.String(), + }, + { + Type: apiv1.StateReady.String(), + Status: metav1.ConditionTrue, + Reason: apiv1.StateReady.String(), + }, + }, + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cr := tt.cr.DeepCopy() + cb := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cr).WithStatusSubresource(cr).WithObjects(tt.objects...).WithStatusSubresource(tt.objects...) + + cliCmd, err := getFakeOrchestratorClient(cr) + require.NoError(t, err) + + r := &PerconaServerMySQLReconciler{ + Client: cb.Build(), + Scheme: scheme, + ClientCmd: cliCmd, + Recorder: new(record.FakeRecorder), + ServerVersion: &platform.ServerVersion{ + Platform: platform.PlatformKubernetes, + }, + } + + cr = &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name, + Namespace: cr.Namespace, + }, + } + + require.NoError(t, r.reconcileCRStatus(ctx, cr, nil)) + require.NoError(t, r.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.Name}, cr)) + + opt := cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "Message") + assert.Empty(t, cmp.Diff(cr.Status, tt.expected, opt)) + }) + } +} + func updateResource[T any](obj *T, updateFuncs ...func(obj *T)) *T { for _, f := range updateFuncs { f(obj) From 17f6aff8b9e104720f600acd89251eea6702afa2 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 27 Mar 2026 11:57:32 +0100 Subject: [PATCH 035/102] make generate manifests main --- deploy/backup/restore.yaml | 1 + deploy/bundle.yaml | 15 +++++++++++++++ deploy/cr.yaml | 15 --------------- deploy/crd.yaml | 15 +++++++++++++++ deploy/cw-bundle.yaml | 15 +++++++++++++++ 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/deploy/backup/restore.yaml b/deploy/backup/restore.yaml index 6790fa2ad..f5be03980 100644 --- a/deploy/backup/restore.yaml +++ b/deploy/backup/restore.yaml @@ -99,3 +99,4 @@ spec: # requests: # storage: 2Gi # storageClassName: standard + pitr: null diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index fcb9c0d56..dc7e78cb3 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -12663,6 +12663,21 @@ spec: properties: backupVersion: type: string + binlogServer: + properties: + imageID: + type: string + ready: + format: int32 + type: integer + size: + format: int32 + type: integer + state: + type: string + version: + type: string + type: object conditions: items: properties: diff --git a/deploy/cr.yaml b/deploy/cr.yaml index bbfbfeaf6..4632fa3a3 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -667,21 +667,6 @@ spec: enabled: true pitr: enabled: false - binlogServer: - size: 1 - image: perconalab/percona-binlog-server:0.2.0 - connectTimeout: 60 - readTimeout: 60 - writeTimeout: 60 - serverId: 42 - idleTime: 60 - storage: - s3: - bucket: S3-BACKUP-BUCKET-NAME-HERE - credentialsSecret: ps-cluster1-s3-credentials - # endpointUrl: https://s3.amazonaws.com - # prefix: PREFIX_NAME - region: us-west-2 # sourcePod: ps-cluster1-mysql-1 image: perconalab/percona-server-mysql-operator:main-backup8.4 imagePullPolicy: Always diff --git a/deploy/crd.yaml b/deploy/crd.yaml index df8820902..cdd2f6c16 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -12663,6 +12663,21 @@ spec: properties: backupVersion: type: string + binlogServer: + properties: + imageID: + type: string + ready: + format: int32 + type: integer + size: + format: int32 + type: integer + state: + type: string + version: + type: string + type: object conditions: items: properties: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index d469cd18b..05cea18a0 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -12663,6 +12663,21 @@ spec: properties: backupVersion: type: string + binlogServer: + properties: + imageID: + type: string + ready: + format: int32 + type: integer + size: + format: int32 + type: integer + state: + type: string + version: + type: string + type: object conditions: items: properties: From a2a2c8ab7920b9741f0b16b9ab4c771f74985b00 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 27 Mar 2026 11:58:19 +0100 Subject: [PATCH 036/102] commit /api --- api/v1/perconaservermysql_types.go | 1 + api/v1/zz_generated.deepcopy.go | 1 + 2 files changed, 2 insertions(+) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index f6ba8183c..dc8189a65 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -634,6 +634,7 @@ type PerconaServerMySQLStatus struct { // INSERT ADDITIONAL STATUS FIELD - defin Orchestrator StatefulAppStatus `json:"orchestrator,omitempty"` HAProxy StatefulAppStatus `json:"haproxy,omitempty"` Router StatefulAppStatus `json:"router,omitempty"` + BinlogServer StatefulAppStatus `json:"binlogServer,omitempty"` State StatefulAppState `json:"state,omitempty"` BackupVersion string `json:"backupVersion,omitempty"` PMMVersion string `json:"pmmVersion,omitempty"` diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index ca874fc37..2e90a3cf9 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -900,6 +900,7 @@ func (in *PerconaServerMySQLStatus) DeepCopyInto(out *PerconaServerMySQLStatus) out.Orchestrator = in.Orchestrator out.HAProxy = in.HAProxy out.Router = in.Router + out.BinlogServer = in.BinlogServer if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]apismetav1.Condition, len(*in)) From f4349db65748ed0db5a8451c0f91563f7aa7e108 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 27 Mar 2026 12:04:57 +0100 Subject: [PATCH 037/102] add /config changes --- .../bases/ps.percona.com_perconaservermysqls.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 8b73d49d8..f596a484e 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -10323,6 +10323,21 @@ spec: properties: backupVersion: type: string + binlogServer: + properties: + imageID: + type: string + ready: + format: int32 + type: integer + size: + format: int32 + type: integer + state: + type: string + version: + type: string + type: object conditions: items: properties: From b17dd59df9410a1165b8618ecda70c5f153cebaa Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 27 Mar 2026 15:20:39 +0100 Subject: [PATCH 038/102] objectKeyFromURI handle for s3 case --- cmd/pitr/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 3142eb0a8..919f304f6 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -237,12 +237,15 @@ func getLatestGTIDByDatetime(relayLogPath, stopDatetime string) (string, error) // objectKeyFromURI extracts the S3 object key from a full URI. // e.g. "https://minio-service:9000/bucket/binlogs/binlog.000001" -> "binlogs/binlog.000001" +// e.g. "s3://bucket/prefix/binlog.000001" -> "prefix/binlog.000001" func objectKeyFromURI(uri, bucket string) (string, error) { u, err := url.Parse(uri) if err != nil { return "", fmt.Errorf("parse URL: %w", err) } - // Path is "//", strip the leading "//" to get the object key + if u.Scheme == "s3" { + return strings.TrimPrefix(u.Path, "/"), nil + } key := strings.TrimPrefix(u.Path, "/"+bucket+"/") return key, nil } From 0544e228c727e772a6b89ce5db4b8c47757f4826 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 27 Mar 2026 15:20:59 +0100 Subject: [PATCH 039/102] e2e test for failover and switchover --- e2e-tests/tests/pitr-minio/02-assert.yaml | 8 +-- .../tests/pitr-minio/02-create-cluster.yaml | 6 +- e2e-tests/tests/pitr-minio/03-write-data.yaml | 4 +- .../tests/pitr-minio/05-write-more-data.yaml | 2 +- e2e-tests/tests/pitr-minio/06-assert.yaml | 2 +- .../pitr-minio/06-create-pitr-restore.yaml | 4 +- e2e-tests/tests/pitr-minio/08-assert.yaml | 2 +- e2e-tests/tests/pitr-minio/10-failover.yaml | 37 +++++++++++ e2e-tests/tests/pitr-minio/11-assert.yaml | 27 ++++++++ .../tests/pitr-minio/11-write-and-backup.yaml | 24 +++++++ e2e-tests/tests/pitr-minio/12-assert.yaml | 27 ++++++++ .../pitr-minio/12-write-and-restore.yaml | 30 +++++++++ e2e-tests/tests/pitr-minio/13-assert.yaml | 24 +++++++ e2e-tests/tests/pitr-minio/13-read-data.yaml | 16 +++++ e2e-tests/tests/pitr-minio/14-switchover.yaml | 64 +++++++++++++++++++ 15 files changed, 263 insertions(+), 14 deletions(-) create mode 100644 e2e-tests/tests/pitr-minio/10-failover.yaml create mode 100644 e2e-tests/tests/pitr-minio/11-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/11-write-and-backup.yaml create mode 100644 e2e-tests/tests/pitr-minio/12-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/12-write-and-restore.yaml create mode 100644 e2e-tests/tests/pitr-minio/13-assert.yaml create mode 100644 e2e-tests/tests/pitr-minio/13-read-data.yaml create mode 100644 e2e-tests/tests/pitr-minio/14-switchover.yaml diff --git a/e2e-tests/tests/pitr-minio/02-assert.yaml b/e2e-tests/tests/pitr-minio/02-assert.yaml index 05e4df507..10149fb52 100644 --- a/e2e-tests/tests/pitr-minio/02-assert.yaml +++ b/e2e-tests/tests/pitr-minio/02-assert.yaml @@ -14,17 +14,15 @@ status: updatedReplicas: 3 collisionCount: 0 --- -kind: StatefulSet +kind: Deployment apiVersion: apps/v1 metadata: - name: pitr-minio-haproxy + name: pitr-minio-router status: observedGeneration: 1 replicas: 3 readyReplicas: 3 - currentReplicas: 3 updatedReplicas: 3 - collisionCount: 0 --- kind: StatefulSet apiVersion: apps/v1 @@ -45,7 +43,7 @@ metadata: finalizers: - percona.com/delete-mysql-pods-in-order status: - haproxy: + router: ready: 3 size: 3 state: ready diff --git a/e2e-tests/tests/pitr-minio/02-create-cluster.yaml b/e2e-tests/tests/pitr-minio/02-create-cluster.yaml index 4ec9c500a..18d12a7af 100644 --- a/e2e-tests/tests/pitr-minio/02-create-cluster.yaml +++ b/e2e-tests/tests/pitr-minio/02-create-cluster.yaml @@ -11,8 +11,9 @@ commands: get_cr \ | yq eval '.spec.mysql.clusterType="group-replication"' - \ | yq eval ".spec.mysql.size=3" - \ - | yq eval ".spec.proxy.haproxy.enabled=true" - \ - | yq eval ".spec.proxy.haproxy.size=3" - \ + | yq eval ".spec.proxy.haproxy.enabled=false" - \ + | yq eval ".spec.proxy.router.enabled=true" - \ + | yq eval ".spec.proxy.router.size=3" - \ | yq eval ".spec.backup.backoffLimit=3" - \ | yq eval '.spec.backup.storages.minio.type="s3"' - \ | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ @@ -27,6 +28,7 @@ commands: | yq eval '.spec.backup.pitr.binlogServer.storage.s3.credentialsSecret="minio-secret"' - \ | yq eval ".spec.backup.pitr.binlogServer.storage.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ | yq eval '.spec.backup.pitr.binlogServer.storage.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.pitr.binlogServer.size=1' - \ | yq eval '.spec.backup.pitr.binlogServer.serverId=100' - \ | yq eval '.spec.backup.pitr.binlogServer.connectTimeout=10' - \ | yq eval '.spec.backup.pitr.binlogServer.readTimeout=10' - \ diff --git a/e2e-tests/tests/pitr-minio/03-write-data.yaml b/e2e-tests/tests/pitr-minio/03-write-data.yaml index d490db677..6bc2f5f72 100644 --- a/e2e-tests/tests/pitr-minio/03-write-data.yaml +++ b/e2e-tests/tests/pitr-minio/03-write-data.yaml @@ -9,8 +9,8 @@ commands: run_mysql \ "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ - "-h $(get_haproxy_svc $(get_cluster_name))" + "-h $(get_router_service $(get_cluster_name))" run_mysql \ "INSERT myDB.myTable (id) VALUES (100500)" \ - "-h $(get_haproxy_svc $(get_cluster_name))" + "-h $(get_router_service $(get_cluster_name))" diff --git a/e2e-tests/tests/pitr-minio/05-write-more-data.yaml b/e2e-tests/tests/pitr-minio/05-write-more-data.yaml index ec30f75d5..31811988b 100644 --- a/e2e-tests/tests/pitr-minio/05-write-more-data.yaml +++ b/e2e-tests/tests/pitr-minio/05-write-more-data.yaml @@ -9,4 +9,4 @@ commands: run_mysql \ "INSERT myDB.myTable (id) VALUES (100501)" \ - "-h $(get_haproxy_svc $(get_cluster_name))" + "-h $(get_router_service $(get_cluster_name))" diff --git a/e2e-tests/tests/pitr-minio/06-assert.yaml b/e2e-tests/tests/pitr-minio/06-assert.yaml index 8f67aa564..22f03ce31 100644 --- a/e2e-tests/tests/pitr-minio/06-assert.yaml +++ b/e2e-tests/tests/pitr-minio/06-assert.yaml @@ -9,7 +9,7 @@ metadata: finalizers: - percona.com/delete-mysql-pods-in-order status: - haproxy: + router: ready: 3 size: 3 state: ready diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml index 52bab6b0e..448bd91b0 100644 --- a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml @@ -12,9 +12,9 @@ commands: run_mysql \ "INSERT myDB.myTable (id) VALUES (100502)" \ - "-h $(get_haproxy_svc ${cluster_name})" + "-h $(get_router_service ${cluster_name})" - PITR_GTID=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_haproxy_svc ${cluster_name})" | tr -d '\n') + PITR_GTID=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_router_service ${cluster_name})" | tr -d '\n') sleep 1 PITR_DATE=$(date -u '+%Y-%m-%d %H:%M:%S') diff --git a/e2e-tests/tests/pitr-minio/08-assert.yaml b/e2e-tests/tests/pitr-minio/08-assert.yaml index e48c39b08..bacd2174b 100644 --- a/e2e-tests/tests/pitr-minio/08-assert.yaml +++ b/e2e-tests/tests/pitr-minio/08-assert.yaml @@ -9,7 +9,7 @@ metadata: finalizers: - percona.com/delete-mysql-pods-in-order status: - haproxy: + router: ready: 3 size: 3 state: ready diff --git a/e2e-tests/tests/pitr-minio/10-failover.yaml b/e2e-tests/tests/pitr-minio/10-failover.yaml new file mode 100644 index 000000000..7f8404385 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/10-failover.yaml @@ -0,0 +1,37 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + primary_before=$(get_primary_from_label) + + kubectl -n "${NAMESPACE}" delete pod "${primary_before}" + + wait_cluster_consistency_gr "${cluster_name}" 3 3 + + primary_after=$(get_primary_from_label) + + if [[ "${primary_before}" == "${primary_after}" ]]; then + echo "Primary pod did not change after failover: was ${primary_before}, still ${primary_after}" + exit 1 + fi + echo "Primary changed from ${primary_before} to ${primary_after}" + + retry=0 + until [[ "$(kubectl -n "${NAMESPACE}" get endpoints "${cluster_name}-mysql-primary" \ + -o jsonpath='{.subsets[0].addresses[0].targetRef.name}' 2>/dev/null)" == "${primary_after}" ]]; do + sleep 5 + retry=$((retry + 1)) + if [ $retry -ge 24 ]; then + echo "Primary service endpoint did not update to ${primary_after} after 2 minutes" + kubectl -n "${NAMESPACE}" get endpoints "${cluster_name}-mysql-primary" -o yaml + exit 1 + fi + done + echo "Primary service endpoint correctly points to new primary: ${primary_after}" diff --git a/e2e-tests/tests/pitr-minio/11-assert.yaml b/e2e-tests/tests/pitr-minio/11-assert.yaml new file mode 100644 index 000000000..df777fbcb --- /dev/null +++ b/e2e-tests/tests/pitr-minio/11-assert.yaml @@ -0,0 +1,27 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + router: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + state: ready +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1 +metadata: + name: pitr-minio-backup-2 +status: + state: Succeeded \ No newline at end of file diff --git a/e2e-tests/tests/pitr-minio/11-write-and-backup.yaml b/e2e-tests/tests/pitr-minio/11-write-and-backup.yaml new file mode 100644 index 000000000..48ce03895 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/11-write-and-backup.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (200500)" \ + "-h $(get_router_service ${cluster_name})" + + echo '{}' \ + | yq eval '.apiVersion = "ps.percona.com/v1"' - \ + | yq eval '.kind = "PerconaServerMySQLBackup"' - \ + | yq eval '.metadata.name = "pitr-minio-backup-2"' - \ + | yq eval '.metadata.finalizers[0] = "percona.com/delete-backup"' - \ + | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ + | yq eval '.spec.storageName = "minio"' - \ + | kubectl apply -n "${NAMESPACE}" -f - \ No newline at end of file diff --git a/e2e-tests/tests/pitr-minio/12-assert.yaml b/e2e-tests/tests/pitr-minio/12-assert.yaml new file mode 100644 index 000000000..b58bd3e0a --- /dev/null +++ b/e2e-tests/tests/pitr-minio/12-assert.yaml @@ -0,0 +1,27 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + router: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + state: ready +--- +kind: PerconaServerMySQLRestore +apiVersion: ps.percona.com/v1 +metadata: + name: pitr-minio-restore-post-failover +status: + state: Succeeded \ No newline at end of file diff --git a/e2e-tests/tests/pitr-minio/12-write-and-restore.yaml b/e2e-tests/tests/pitr-minio/12-write-and-restore.yaml new file mode 100644 index 000000000..d69a9c375 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/12-write-and-restore.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (200501)" \ + "-h $(get_router_service ${cluster_name})" + + sleep 120 + + PITR_DATE_POST=$(date -u '+%Y-%m-%d %H:%M:%S') + kubectl create configmap -n "${NAMESPACE}" pitr-date-post --from-literal=date="${PITR_DATE_POST}" + + echo '{}' \ + | yq eval '.apiVersion = "ps.percona.com/v1"' - \ + | yq eval '.kind = "PerconaServerMySQLRestore"' - \ + | yq eval '.metadata.name = "pitr-minio-restore-post-failover"' - \ + | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ + | yq eval '.spec.backupName = "pitr-minio-backup-2"' - \ + | yq eval '.spec.pitr.type = "date"' - \ + | yq eval ".spec.pitr.date = \"${PITR_DATE_POST}\"" - \ + | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/pitr-minio/13-assert.yaml b/e2e-tests/tests/pitr-minio/13-assert.yaml new file mode 100644 index 000000000..a771c8bac --- /dev/null +++ b/e2e-tests/tests/pitr-minio/13-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-data-0 +data: + max_id: "200501" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-data-1 +data: + max_id: "200501" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-data-2 +data: + max_id: "200501" diff --git a/e2e-tests/tests/pitr-minio/13-read-data.yaml b/e2e-tests/tests/pitr-minio/13-read-data.yaml new file mode 100644 index 000000000..1034b3210 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/13-read-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + for i in 0 1 2; do + max_id=$(run_mysql "SELECT MAX(id) FROM myDB.myTable" "-h ${cluster_name}-mysql-${i}.${cluster_name}-mysql") + kubectl create configmap -n "${NAMESPACE}" 13-read-data-${i} --from-literal=max_id="${max_id}" + done diff --git a/e2e-tests/tests/pitr-minio/14-switchover.yaml b/e2e-tests/tests/pitr-minio/14-switchover.yaml new file mode 100644 index 000000000..899a70d31 --- /dev/null +++ b/e2e-tests/tests/pitr-minio/14-switchover.yaml @@ -0,0 +1,64 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + primary_before=$(get_primary_from_label) + + new_primary="" + for i in 0 1 2; do + pod="${cluster_name}-mysql-${i}" + if [[ "${pod}" != "${primary_before}" ]]; then + new_primary="${pod}" + break + fi + done + + if [[ -z "${new_primary}" ]]; then + echo "Could not find a secondary pod to switchover to" + exit 1 + fi + + new_primary_fqdn="${new_primary}.${cluster_name}-mysql.${NAMESPACE}:3306" + echo "Switching primary from ${primary_before} to ${new_primary} (${new_primary_fqdn})" + + uri=$(get_mysqlsh_uri_for_pod "${primary_before}") + client_pod=$(get_client_pod) + wait_pod "${client_pod}" 1>&2 + + kubectl -n "${NAMESPACE}" exec "${client_pod}" -- \ + mysqlsh --js --quiet-start=2 --uri "${uri}" -- cluster setPrimary "${new_primary_fqdn}" + + wait_cluster_consistency_gr "${cluster_name}" 3 3 + + primary_after=$(get_primary_from_label) + + if [[ "${primary_before}" == "${primary_after}" ]]; then + echo "Primary pod did not change after switchover: was ${primary_before}, still ${primary_after}" + exit 1 + fi + + if [[ "${primary_after}" != "${new_primary}" ]]; then + echo "Primary after switchover (${primary_after}) does not match expected (${new_primary})" + exit 1 + fi + echo "Primary changed from ${primary_before} to ${primary_after}" + + retry=0 + until [[ "$(kubectl -n "${NAMESPACE}" get endpoints "${cluster_name}-primary" \ + -o jsonpath='{.subsets[0].addresses[0].targetRef.name}' 2>/dev/null)" == "${primary_after}" ]]; do + sleep 5 + retry=$((retry + 1)) + if [ $retry -ge 24 ]; then + echo "Primary service endpoint did not update to ${primary_after} after 2 minutes" + kubectl -n "${NAMESPACE}" get endpoints "${cluster_name}-primary" -o yaml + exit 1 + fi + done + echo "Primary service endpoint correctly points to new primary after switchover: ${primary_after}" From 351dbba9ee6d8d62c875c65535cdfd38356f725e Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 27 Mar 2026 20:04:16 +0100 Subject: [PATCH 040/102] e2e test fixes --- e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml | 8 +++++++- e2e-tests/tests/pitr-minio/07-assert.yaml | 6 +++--- e2e-tests/tests/pitr-minio/09-assert.yaml | 6 +++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml index 448bd91b0..c111d8855 100644 --- a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml @@ -10,11 +10,17 @@ commands: cluster_name=$(get_cluster_name) + GTID_BEFORE=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_router_service ${cluster_name})" | tr -d '\n') + run_mysql \ "INSERT myDB.myTable (id) VALUES (100502)" \ "-h $(get_router_service ${cluster_name})" - PITR_GTID=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_router_service ${cluster_name})" | tr -d '\n') + PITR_GTID=$(run_mysql "SELECT GTID_SUBTRACT(@@GLOBAL.gtid_executed, '${GTID_BEFORE}')" "-h $(get_router_service ${cluster_name})" | tr -d '\n') + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100503)" \ + "-h $(get_router_service ${cluster_name})" sleep 1 PITR_DATE=$(date -u '+%Y-%m-%d %H:%M:%S') diff --git a/e2e-tests/tests/pitr-minio/07-assert.yaml b/e2e-tests/tests/pitr-minio/07-assert.yaml index 75528883a..bbd6bf1cc 100644 --- a/e2e-tests/tests/pitr-minio/07-assert.yaml +++ b/e2e-tests/tests/pitr-minio/07-assert.yaml @@ -7,18 +7,18 @@ apiVersion: v1 metadata: name: 07-read-data-0 data: - max_id: "100500" + max_id: "100501" --- kind: ConfigMap apiVersion: v1 metadata: name: 07-read-data-1 data: - max_id: "100500" + max_id: "100501" --- kind: ConfigMap apiVersion: v1 metadata: name: 07-read-data-2 data: - max_id: "100500" + max_id: "100501" diff --git a/e2e-tests/tests/pitr-minio/09-assert.yaml b/e2e-tests/tests/pitr-minio/09-assert.yaml index 58b18699b..5a75166e4 100644 --- a/e2e-tests/tests/pitr-minio/09-assert.yaml +++ b/e2e-tests/tests/pitr-minio/09-assert.yaml @@ -7,18 +7,18 @@ apiVersion: v1 metadata: name: 09-read-data-0 data: - max_id: "100501" + max_id: "100502" --- kind: ConfigMap apiVersion: v1 metadata: name: 09-read-data-1 data: - max_id: "100501" + max_id: "100502" --- kind: ConfigMap apiVersion: v1 metadata: name: 09-read-data-2 data: - max_id: "100501" + max_id: "100502" From d075fe6327d6100196a05597df0f89336c3cddb4 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 27 Mar 2026 22:44:29 +0100 Subject: [PATCH 041/102] add cr options comments for pirt --- deploy/cr.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 4632fa3a3..29a8cfb0f 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -667,6 +667,20 @@ spec: enabled: true pitr: enabled: false +# binlogServer: +# size: 1 +# serverId: 100 +# storage: +# s3: +# bucket: S3-BACKUP-BUCKET-NAME-HERE +# credentialsSecret: ps-cluster1-s3-credentials +# endpointUrl: https://s3.amazonaws.com +# prefix: PREFIX_NAME +# region: us-west-2 +# connectTimeout: 10 +# readTimeout: 10 +# writeTimeout: 10 +# idleTime: 3 # sourcePod: ps-cluster1-mysql-1 image: perconalab/percona-server-mysql-operator:main-backup8.4 imagePullPolicy: Always From a4c1e46cf6479fc5cd158e227b465e743a59793c Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 31 Mar 2026 13:29:09 +0300 Subject: [PATCH 042/102] fix assert for date --- e2e-tests/tests/pitr-minio/09-assert.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e-tests/tests/pitr-minio/09-assert.yaml b/e2e-tests/tests/pitr-minio/09-assert.yaml index 5a75166e4..8f6ce88d6 100644 --- a/e2e-tests/tests/pitr-minio/09-assert.yaml +++ b/e2e-tests/tests/pitr-minio/09-assert.yaml @@ -7,18 +7,18 @@ apiVersion: v1 metadata: name: 09-read-data-0 data: - max_id: "100502" + max_id: "100503" --- kind: ConfigMap apiVersion: v1 metadata: name: 09-read-data-1 data: - max_id: "100502" + max_id: "100503" --- kind: ConfigMap apiVersion: v1 metadata: name: 09-read-data-2 data: - max_id: "100502" + max_id: "100503" From 7761dba918961e571f0e4af56eec7372c08989eb Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 31 Mar 2026 13:38:04 +0300 Subject: [PATCH 043/102] use start and stop datetime for date pitr restore --- cmd/pitr/main.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 919f304f6..80dad56a8 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -209,10 +209,16 @@ func runApply(ctx context.Context) error { return nil } -func getLatestGTIDByDatetime(relayLogPath, stopDatetime string) (string, error) { +func getLatestGTIDByDatetime(relayLogPath, startDatetime string) (string, error) { + t, err := time.ParseInLocation("2006-01-02 15:04:05", startDatetime, time.UTC) + if err != nil { + return "", fmt.Errorf("parse datetime %q: %w", startDatetime, err) + } + stopDatetime := t.Add(time.Second).Format("2006-01-02 15:04:05") + cmd := exec.Command("bash", "-c", - fmt.Sprintf("mysqlbinlog --stop-datetime='%s' %s | grep GTID_NEXT | grep -v AUTOMATIC | tail -n 1", - stopDatetime, relayLogPath)) + fmt.Sprintf("mysqlbinlog --start-datetime='%s' --stop-datetime='%s' %s | grep GTID_NEXT | grep -v AUTOMATIC | head -n 1", + startDatetime, stopDatetime, relayLogPath)) output, err := cmd.Output() if err != nil { @@ -221,7 +227,7 @@ func getLatestGTIDByDatetime(relayLogPath, stopDatetime string) (string, error) line := strings.TrimSpace(string(output)) if line == "" { - return "", fmt.Errorf("no GTID found before %s in %s", stopDatetime, relayLogPath) + return "", fmt.Errorf("no GTID found at %s in %s", startDatetime, relayLogPath) } // Extract GTID from: SET @@SESSION.GTID_NEXT= 'uuid:n,uuid:n'/*!*/; From 32b7646e678935ff79f571a526f2bcad3510dd5e Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 31 Mar 2026 13:45:54 +0300 Subject: [PATCH 044/102] set error when binlogs cannot be collected and gtid next to automatic at the end of the restore process --- cmd/internal/db/db.go | 5 +++++ cmd/pitr/main.go | 5 +++++ pkg/controller/psrestore/controller.go | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cmd/internal/db/db.go b/cmd/internal/db/db.go index f490dc2dc..dfc052892 100644 --- a/cmd/internal/db/db.go +++ b/cmd/internal/db/db.go @@ -420,6 +420,11 @@ func (d *DB) StartReplicaUntilPosition(ctx context.Context, relayLogFile string, return errors.Wrap(err, "start replica until position") } +func (d *DB) SetGTIDNextAutomatic(ctx context.Context) error { + _, err := d.db.ExecContext(ctx, "SET GTID_NEXT='AUTOMATIC'") + return errors.Wrap(err, "set GTID_NEXT to AUTOMATIC") +} + func (d *DB) WaitReplicaSQLThreadStop(ctx context.Context, pollInterval time.Duration) error { for { var serviceState string diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 80dad56a8..02b37f1b1 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -205,6 +205,11 @@ func runApply(ctx context.Context) error { return errors.Wrap(err, "reset replication") } + log.Println("setting GTID_NEXT to AUTOMATIC") + if err := database.SetGTIDNextAutomatic(ctx); err != nil { + return fmt.Errorf("set GTID_NEXT to AUTOMATIC: %w", err) + } + log.Println("PITR apply complete") return nil } diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index ada42340c..f964a46c5 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -183,8 +183,9 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req if cr.Spec.PITR != nil { status.State = apiv1.RestoreStarting if err := r.reconcilePITRConfig(ctx, cr, cluster); err != nil { + status.State = apiv1.RestoreError status.StateDesc = errors.Wrap(err, "reconcile pitr config").Error() - return ctrl.Result{}, errors.Wrap(err, "reconcile pitr config") + return ctrl.Result{}, nil } status.StateDesc = "" } From 54bd5428c559bbfa2e96ec32bb61def11d4b7902 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 31 Mar 2026 19:52:46 +0300 Subject: [PATCH 045/102] fixes on datetime gitd --- cmd/pitr/main.go | 4 ++-- e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 02b37f1b1..898a2e228 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -222,8 +222,8 @@ func getLatestGTIDByDatetime(relayLogPath, startDatetime string) (string, error) stopDatetime := t.Add(time.Second).Format("2006-01-02 15:04:05") cmd := exec.Command("bash", "-c", - fmt.Sprintf("mysqlbinlog --start-datetime='%s' --stop-datetime='%s' %s | grep GTID_NEXT | grep -v AUTOMATIC | head -n 1", - startDatetime, stopDatetime, relayLogPath)) + fmt.Sprintf("mysqlbinlog --stop-datetime='%s' %s | grep GTID_NEXT | grep -v AUTOMATIC | tail -n 1", + stopDatetime, relayLogPath)) output, err := cmd.Output() if err != nil { diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml index c111d8855..eba94939f 100644 --- a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml @@ -22,7 +22,7 @@ commands: "INSERT myDB.myTable (id) VALUES (100503)" \ "-h $(get_router_service ${cluster_name})" - sleep 1 + sleep 10 PITR_DATE=$(date -u '+%Y-%m-%d %H:%M:%S') kubectl create configmap -n "${NAMESPACE}" pitr-date --from-literal=date="${PITR_DATE}" From 37f23328f41ce6b65fa3a760b5ac80f70e7c4202 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Wed, 1 Apr 2026 18:07:29 +0300 Subject: [PATCH 046/102] fixes on fields and crd --- api/v1/perconaservermysql_types.go | 14 ++-- api/v1/perconaservermysqlrestore_types.go | 1 + cmd/example-gen/main.go | 5 ++ cmd/example-gen/pkg/defaults/manual.go | 21 +++++ cmd/example-gen/pkg/defaults/values.go | 1 + cmd/example-gen/scripts/lib/ps-restore.sh | 3 +- cmd/example-gen/scripts/lib/ps.sh | 17 +++- ...ercona.com_perconaservermysqlrestores.yaml | 3 + .../ps.percona.com_perconaservermysqls.yaml | 9 +-- deploy/backup/restore.yaml | 5 +- deploy/bundle.yaml | 12 +-- deploy/cr.yaml | 81 +++++++++++++++++++ deploy/crd.yaml | 12 +-- deploy/cw-bundle.yaml | 12 +-- 14 files changed, 160 insertions(+), 36 deletions(-) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index dc8189a65..aff6d8fae 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -512,19 +512,20 @@ type BinlogServerStorageSpec struct { S3 *BackupStorageS3Spec `json:"s3,omitempty"` } +// +kubebuilder:validation:XValidation:rule="self.size <= 1",message="binlogServer size cannot be more than 1" type BinlogServerSpec struct { - Storage BinlogServerStorageSpec `json:"storage"` + Storage BinlogServerStorageSpec `json:"storage,omitempty"` // The number of seconds the MySQL client library will wait to establish a connection with a remote host - ConnectTimeout int32 `json:"connectTimeout"` + ConnectTimeout int32 `json:"connectTimeout,omitempty"` // The number of seconds the MySQL client library will wait to read data from a remote server. - ReadTimeout int32 `json:"readTimeout"` + ReadTimeout int32 `json:"readTimeout,omitempty"` // The number of seconds the MySQL client library will wait to write data to a remote server. - WriteTimeout int32 `json:"writeTimeout"` + WriteTimeout int32 `json:"writeTimeout,omitempty"` // Specifies the server ID that the utility will be using when connecting to a remote MySQL server - ServerID int32 `json:"serverId"` + ServerID int32 `json:"serverId,omitempty"` // The number of seconds the utility will spend in disconnected mode between reconnection attempts. - IdleTime int32 `json:"idleTime"` + IdleTime int32 `json:"idleTime,omitempty"` PodSpec `json:",inline"` } @@ -1259,5 +1260,4 @@ const ( UpgradeStrategyDisabled = "disabled" UpgradeStrategyNever = "never" UpgradeStrategyRecommended = "recommended" - UpgradeStrategyLatest = "latest" ) diff --git a/api/v1/perconaservermysqlrestore_types.go b/api/v1/perconaservermysqlrestore_types.go index de7c824e1..da4a7dc4e 100644 --- a/api/v1/perconaservermysqlrestore_types.go +++ b/api/v1/perconaservermysqlrestore_types.go @@ -35,6 +35,7 @@ type PerconaServerMySQLRestoreSpec struct { } type RestorePITRSpec struct { + // +kubebuilder:validation:Enum=gtid;date Type PITRType `json:"type"` Date string `json:"date,omitempty"` GTID string `json:"gtid,omitempty"` diff --git a/cmd/example-gen/main.go b/cmd/example-gen/main.go index 70dba8dd7..b77971ee2 100644 --- a/cmd/example-gen/main.go +++ b/cmd/example-gen/main.go @@ -161,6 +161,11 @@ func printRestore() error { Spec: apiv1.PerconaServerMySQLRestoreSpec{ ClusterName: defaults.NameCluster, BackupName: defaults.NameBackup, + PITR: &apiv1.RestorePITRSpec{ + Type: apiv1.PITRDate, + Date: "2024-11-18T11:10:48Z", + GTID: "a3e5ff70-83e2-11ef-8e57-7a62caf7e1e3:1-36", + }, BackupSource: &apiv1.PerconaServerMySQLBackupStatus{ Destination: "s3://S3-BACKUP-BUCKET-NAME-HERE/backup-path", Storage: &apiv1.BackupStorageSpec{ diff --git a/cmd/example-gen/pkg/defaults/manual.go b/cmd/example-gen/pkg/defaults/manual.go index 9c3ee45dd..a113dc7d1 100644 --- a/cmd/example-gen/pkg/defaults/manual.go +++ b/cmd/example-gen/pkg/defaults/manual.go @@ -121,6 +121,27 @@ func pmmDefaults(spec *apiv1.PMMSpec) { func backupDefaults(spec *apiv1.BackupSpec) { spec.Image = ImageBackup spec.Enabled = true + spec.PiTR = apiv1.PiTRSpec{ + Enabled: false, + BinlogServer: &apiv1.BinlogServerSpec{ + Storage: apiv1.BinlogServerStorageSpec{ + S3: &apiv1.BackupStorageS3Spec{ + Bucket: "S3-BACKUP-BUCKET-NAME-HERE", + Prefix: "PREFIX_NAME", + CredentialsSecret: fmt.Sprintf("%s-s3-credentials", NameCluster), + Region: "us-west-2", + EndpointURL: "https://s3.amazonaws.com", + }, + }, + ConnectTimeout: 10, + ReadTimeout: 10, + WriteTimeout: 10, + ServerID: 100, + IdleTime: 3, + }, + } + podSpecDefaults(&spec.PiTR.BinlogServer.PodSpec, ImageBinlogServer, corev1.ResourceRequirements{}, "", 30, nil, nil) + spec.PiTR.BinlogServer.Size = 1 spec.SourcePod = SourcePod spec.ServiceAccountName = "some-service-account" spec.BackoffLimit = ptr.To(int32(6)) diff --git a/cmd/example-gen/pkg/defaults/values.go b/cmd/example-gen/pkg/defaults/values.go index 1160c836e..44205bf96 100644 --- a/cmd/example-gen/pkg/defaults/values.go +++ b/cmd/example-gen/pkg/defaults/values.go @@ -16,6 +16,7 @@ const ( ImageOrchestrator = "perconalab/percona-server-mysql-operator:main-orchestrator" ImagePMM = "perconalab/pmm-client:3-dev-latest" ImageBackup = "perconalab/percona-server-mysql-operator:main-backup8.4" + ImageBinlogServer = "perconalab/percona-binlog-server:0.2.0" ImageToolkit = "perconalab/percona-server-mysql-operator:main-toolkit" ) diff --git a/cmd/example-gen/scripts/lib/ps-restore.sh b/cmd/example-gen/scripts/lib/ps-restore.sh index cf7ef2a54..d11972ecb 100644 --- a/cmd/example-gen/scripts/lib/ps-restore.sh +++ b/cmd/example-gen/scripts/lib/ps-restore.sh @@ -6,7 +6,7 @@ export RESOURCE_PATH="deploy/backup/restore.yaml" sort_yaml() { - SPEC_ORDER='"clusterName", "backupName", "containerOptions", "backupSource"' + SPEC_ORDER='"clusterName", "backupName", "pitr", "containerOptions", "backupSource"' CONTAINER_OPTS_ORDER='"env", "args"' yq - \ @@ -28,6 +28,7 @@ remove_fields() { del_fields_to_comment() { yq - \ + | yq "del(.spec.pitr)" \ | yq "del(.spec.containerOptions)" \ | yq "del(.spec.backupSource)" } diff --git a/cmd/example-gen/scripts/lib/ps.sh b/cmd/example-gen/scripts/lib/ps.sh index 0b1b80b90..b38e1d079 100644 --- a/cmd/example-gen/scripts/lib/ps.sh +++ b/cmd/example-gen/scripts/lib/ps.sh @@ -15,7 +15,9 @@ sort_yaml() { ORCHESTRATOR_ORDER='"enabled", "expose", '"$POD_SPEC_ORDER" PMM_ORDER='"enabled","image","imagePullPolicy","serverHost","mysqlParams","containerSecurityContext", "resources", "readinessProbes", "livenessProbes"' - BACKUP_ORDER='"enabled","pitr","sourcePod","image","imagePullPolicy","imagePullSecrets","schedule","backoffLimit", "serviceAccountName", "initContainer", "containerSecurityContext", "resources","storages","pitr"' + BINLOG_SERVER_ORDER='"enabled","binlogServer"' + BINLOG_SERVER_SPEC_ORDER='"size","image","imagePullPolicy","imagePullSecrets","serverId","storage","connectTimeout","readTimeout","writeTimeout","idleTime"' + BACKUP_ORDER='"enabled","pitr","sourcePod","image","imagePullPolicy","imagePullSecrets","schedule","backoffLimit", "serviceAccountName", "initContainer", "containerSecurityContext", "resources","storages"' TOOLKIT_ORDER='"image","imagePullPolicy","imagePullSecrets","env","envFrom","resources","containerSecurityContext", "startupProbe", "readinessProbe", "livenessProbe"' yq - \ @@ -26,12 +28,13 @@ sort_yaml() { | yq '.spec.orchestrator |= pick((['"$ORCHESTRATOR_ORDER"'] + keys) | unique)' \ | yq '.spec.pmm |= pick((['"$PMM_ORDER"'] + keys) | unique)' \ | yq '.spec.backup |= pick((['"$BACKUP_ORDER"'] + keys) | unique)' \ + | yq '.spec.backup.pitr |= pick((['"$BINLOG_SERVER_ORDER"'] + keys) | unique)' \ + | yq '.spec.backup.pitr.binlogServer |= pick((['"$BINLOG_SERVER_SPEC_ORDER"'] + keys) | unique)' \ | yq '.spec.toolkit |= pick((['"$TOOLKIT_ORDER"'] + keys) | unique)' } remove_fields() { # - removing initImage as it is deprecated - # - removing binlogServer is not used # - removing azure-blob fields to reduce size # - removing gcp-cs fields to reduce size # - removing non-s3 fields in s3-us-west @@ -43,7 +46,14 @@ remove_fields() { | yq 'del(.spec.orchestrator.initImage)' \ | yq 'del(.spec.proxy.haproxy.initImage)' \ | yq 'del(.spec.proxy.router.initImage)' \ - | yq 'del(.spec.backup.pitr.binlogServer)' \ + | yq 'del(.spec.backup.pitr.binlogServer.runtimeClassName)' \ + | yq 'del(.spec.backup.pitr.binlogServer.labels)' \ + | yq 'del(.spec.backup.pitr.binlogServer.annotations)' \ + | yq 'del(.spec.backup.pitr.binlogServer.nodeSelector)' \ + | yq 'del(.spec.backup.pitr.binlogServer.priorityClassName)' \ + | yq 'del(.spec.backup.pitr.binlogServer.schedulerName)' \ + | yq 'del(.spec.backup.pitr.binlogServer.serviceAccountName)' \ + | yq 'del(.spec.backup.pitr.binlogServer.gracePeriod)' \ | yq 'del(.spec.backup.storages.azure-blob.affinity)' \ | yq 'del(.spec.backup.storages.azure-blob.annotations)' \ | yq 'del(.spec.backup.storages.azure-blob.gcs)' \ @@ -189,6 +199,7 @@ del_fields_to_comment() { | yq "del(.spec.pmm.livenessProbes)" \ | yq "del(.spec.pmm.containerSecurityContext)" \ | yq "del(.spec.pmm.resources.limits)" \ + | yq "del(.spec.backup.pitr.binlogServer)" \ | yq "del(.spec.backup.sourcePod)" \ | yq "del(.spec.backup.schedule)" \ | yq "del(.spec.backup.backoffLimit)" \ diff --git a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml index 8f3e987e0..e02042aca 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml @@ -1143,6 +1143,9 @@ spec: gtid: type: string type: + enum: + - gtid + - date type: string required: - type diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index f596a484e..7fccdd020 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -1511,15 +1511,12 @@ spec: format: int32 type: integer required: - - connectTimeout - - idleTime - image - - readTimeout - - serverId - size - - storage - - writeTimeout type: object + x-kubernetes-validations: + - message: binlogServer size cannot be more than 1 + rule: self.size <= 1 enabled: type: boolean type: object diff --git a/deploy/backup/restore.yaml b/deploy/backup/restore.yaml index f5be03980..7622a6f8e 100644 --- a/deploy/backup/restore.yaml +++ b/deploy/backup/restore.yaml @@ -5,6 +5,10 @@ metadata: spec: clusterName: ps-cluster1 backupName: backup1 +# pitr: +# date: "2024-11-18T11:10:48Z" +# gtid: a3e5ff70-83e2-11ef-8e57-7a62caf7e1e3:1-36 +# type: date # containerOptions: # env: # - name: CUSTOM_VAR @@ -99,4 +103,3 @@ spec: # requests: # storage: 2Gi # storageClassName: standard - pitr: null diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index dc7e78cb3..c447b9c06 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -2311,6 +2311,9 @@ spec: gtid: type: string type: + enum: + - gtid + - date type: string required: - type @@ -3851,15 +3854,12 @@ spec: format: int32 type: integer required: - - connectTimeout - - idleTime - image - - readTimeout - - serverId - size - - storage - - writeTimeout type: object + x-kubernetes-validations: + - message: binlogServer size cannot be more than 1 + rule: self.size <= 1 enabled: type: boolean type: object diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 29a8cfb0f..debf93c92 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -669,6 +669,11 @@ spec: enabled: false # binlogServer: # size: 1 +# image: perconalab/percona-binlog-server:0.2.0 +# imagePullPolicy: Always +# imagePullSecrets: +# - name: my-secret-1 +# - name: my-secret-2 # serverId: 100 # storage: # s3: @@ -681,6 +686,82 @@ spec: # readTimeout: 10 # writeTimeout: 10 # idleTime: 3 +# affinity: +# advanced: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: kubernetes.io/e2e-az-name +# operator: In +# values: +# - e2e-az1 +# - e2e-az2 +# antiAffinityTopologyKey: kubernetes.io/hostname +# configuration: "" +# containerSecurityContext: +# privileged: false +# runAsGroup: 1001 +# runAsUser: 1001 +# env: [] +# envFrom: [] +# initContainer: +# containerSecurityContext: +# privileged: false +# runAsGroup: 1001 +# runAsUser: 1001 +# image: perconalab/percona-server-mysql-operator:main +# resources: +# limits: +# cpu: 100m +# memory: 100M +# requests: +# cpu: 200m +# memory: 200M +# initImage: "" +# livenessProbe: +# failureThreshold: 3 +# periodSeconds: 5 +# successThreshold: 1 +# timeoutSeconds: 3 +# podDisruptionBudget: +# maxUnavailable: 1 +# minAvailable: 0 +# podSecurityContext: +# fsGroup: 1001 +# supplementalGroups: +# - 1001 +# - 1002 +# - 1003 +# readinessProbe: +# failureThreshold: 3 +# periodSeconds: 5 +# successThreshold: 1 +# timeoutSeconds: 3 +# resources: +# limits: +# cpu: 100m +# memory: 100M +# requests: +# cpu: 200m +# memory: 200M +# startupProbe: +# failureThreshold: 3 +# periodSeconds: 5 +# successThreshold: 1 +# timeoutSeconds: 3 +# tolerations: +# - effect: NoExecute +# key: node.alpha.kubernetes.io/unreachable +# operator: Exists +# tolerationSeconds: 6000 +# topologySpreadConstraints: +# - labelSelector: +# matchLabels: +# app.kubernetes.io/name: percona-server +# maxSkew: 1 +# topologyKey: kubernetes.io/hostname +# whenUnsatisfiable: DoNotSchedule # sourcePod: ps-cluster1-mysql-1 image: perconalab/percona-server-mysql-operator:main-backup8.4 imagePullPolicy: Always diff --git a/deploy/crd.yaml b/deploy/crd.yaml index cdd2f6c16..e0962baf0 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -2311,6 +2311,9 @@ spec: gtid: type: string type: + enum: + - gtid + - date type: string required: - type @@ -3851,15 +3854,12 @@ spec: format: int32 type: integer required: - - connectTimeout - - idleTime - image - - readTimeout - - serverId - size - - storage - - writeTimeout type: object + x-kubernetes-validations: + - message: binlogServer size cannot be more than 1 + rule: self.size <= 1 enabled: type: boolean type: object diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 05cea18a0..660eb12d1 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -2311,6 +2311,9 @@ spec: gtid: type: string type: + enum: + - gtid + - date type: string required: - type @@ -3851,15 +3854,12 @@ spec: format: int32 type: integer required: - - connectTimeout - - idleTime - image - - readTimeout - - serverId - size - - storage - - writeTimeout type: object + x-kubernetes-validations: + - message: binlogServer size cannot be more than 1 + rule: self.size <= 1 enabled: type: boolean type: object From e97a5c1eb1990de5e7e77bb628d1f30a27289fab Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Wed, 1 Apr 2026 23:39:10 +0300 Subject: [PATCH 047/102] handle validation rules for size and image, fix for binlog but affect all components --- api/v1/perconaservermysql_types.go | 20 +- .../ps.percona.com_perconaservermysqls.yaml | 67 +++++-- deploy/bundle.yaml | 67 +++++-- deploy/crd.yaml | 67 +++++-- deploy/cw-bundle.yaml | 67 +++++-- pkg/controller/ps/controller_test.go | 174 +++++++++++++++++- 6 files changed, 374 insertions(+), 88 deletions(-) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index aff6d8fae..5cc2b91be 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -135,6 +135,8 @@ func (t ClusterType) isValid() bool { return false } +// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ''",message="mysql.image is required" +// +kubebuilder:validation:XValidation:rule="has(self.size) && self.size > 0",message="mysql.size must be greater than 0" type MySQLSpec struct { ClusterType ClusterType `json:"clusterType,omitempty"` ExposePrimary ServiceExposeTogglable `json:"exposePrimary,omitempty"` @@ -168,6 +170,8 @@ type SidecarPVC struct { Spec corev1.PersistentVolumeClaimSpec `json:"spec"` } +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="orchestrator.image is required when orchestrator is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="orchestrator.size must be greater than 0 when orchestrator is enabled" type OrchestratorSpec struct { Enabled bool `json:"enabled,omitempty"` Expose ServiceExpose `json:"expose,omitempty"` @@ -176,7 +180,7 @@ type OrchestratorSpec struct { } type ContainerSpec struct { - Image string `json:"image"` + Image string `json:"image,omitempty"` ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` Resources corev1.ResourceRequirements `json:"resources,omitempty"` @@ -192,7 +196,6 @@ type ContainerSpec struct { } type PodSpec struct { - // +kubebuilder:validation:Required Size int32 `json:"size,omitempty"` Annotations map[string]string `json:"annotations,omitempty"` Labels map[string]string `json:"labels,omitempty"` @@ -278,9 +281,10 @@ func (s *PodSpec) GetInitSpec(cr *PerconaServerMySQL) InitContainerSpec { return *s.InitContainer } +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="pmm.image is required when pmm is enabled" type PMMSpec struct { Enabled bool `json:"enabled,omitempty"` - Image string `json:"image"` + Image string `json:"image,omitempty"` MySQLParams string `json:"mysqlParams,omitempty"` ServerHost string `json:"serverHost,omitempty"` Resources corev1.ResourceRequirements `json:"resources,omitempty"` @@ -502,6 +506,9 @@ func (b *BackupStorageAzureSpec) ContainerAndPrefix() (string, string) { return container, prefix } +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || has(self.binlogServer)",message="binlogServer is required when pitr is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image != '')",message="binlogServer.image is required when pitr is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size > 0)",message="binlogServer.size is required when pitr is enabled" type PiTRSpec struct { Enabled bool `json:"enabled,omitempty"` @@ -512,7 +519,7 @@ type BinlogServerStorageSpec struct { S3 *BackupStorageS3Spec `json:"s3,omitempty"` } -// +kubebuilder:validation:XValidation:rule="self.size <= 1",message="binlogServer size cannot be more than 1" +// +kubebuilder:validation:XValidation:rule="!has(self.size) || self.size <= 1",message="binlogServer size cannot be more than 1" type BinlogServerSpec struct { Storage BinlogServerStorageSpec `json:"storage,omitempty"` @@ -535,6 +542,8 @@ type ProxySpec struct { HAProxy *HAProxySpec `json:"haproxy,omitempty"` } +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="router.image is required when router is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="router.size must be greater than 0 when router is enabled" type MySQLRouterSpec struct { Enabled bool `json:"enabled,omitempty"` @@ -545,10 +554,13 @@ type MySQLRouterSpec struct { PodSpec `json:",inline"` } +// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ''",message="toolkit.image is required" type ToolkitSpec struct { ContainerSpec `json:",inline"` } +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="haproxy.image is required when haproxy is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="haproxy.size must be greater than 0 when haproxy is enabled" type HAProxySpec struct { Enabled bool `json:"enabled,omitempty"` diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 7fccdd020..83dad6409 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -1510,16 +1510,24 @@ spec: writeTimeout: format: int32 type: integer - required: - - image - - size type: object x-kubernetes-validations: - message: binlogServer size cannot be more than 1 - rule: self.size <= 1 + rule: '!has(self.size) || self.size <= 1' enabled: type: boolean type: object + x-kubernetes-validations: + - message: binlogServer is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' + - message: binlogServer.image is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.image) && self.binlogServer.image + != '''')' + - message: binlogServer.size is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.size) && self.binlogServer.size + > 0)' resources: properties: claims: @@ -5696,10 +5704,12 @@ spec: type: string type: object type: object - required: - - image - - size type: object + x-kubernetes-validations: + - message: mysql.image is required + rule: has(self.image) && self.image != '' + - message: mysql.size must be greater than 0 + rule: has(self.size) && self.size > 0 orchestrator: properties: affinity: @@ -6940,10 +6950,15 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: orchestrator.image is required when orchestrator is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: orchestrator.size must be greater than 0 when orchestrator + is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && + self.size > 0)' pause: type: boolean pmm: @@ -7225,9 +7240,11 @@ spec: type: object serverHost: type: string - required: - - image type: object + x-kubernetes-validations: + - message: pmm.image is required when pmm is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' proxy: properties: haproxy: @@ -8470,10 +8487,15 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: haproxy.image is required when haproxy is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: haproxy.size must be greater than 0 when haproxy is + enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) + && self.size > 0)' router: properties: affinity: @@ -9739,10 +9761,14 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: router.image is required when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: router.size must be greater than 0 when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) + && self.size > 0)' type: object secretsName: type: string @@ -10237,9 +10263,10 @@ spec: format: int32 type: integer type: object - required: - - image type: object + x-kubernetes-validations: + - message: toolkit.image is required + rule: has(self.image) && self.image != '' unsafeFlags: properties: mysqlSize: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index c447b9c06..414f6457b 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -3853,16 +3853,24 @@ spec: writeTimeout: format: int32 type: integer - required: - - image - - size type: object x-kubernetes-validations: - message: binlogServer size cannot be more than 1 - rule: self.size <= 1 + rule: '!has(self.size) || self.size <= 1' enabled: type: boolean type: object + x-kubernetes-validations: + - message: binlogServer is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' + - message: binlogServer.image is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.image) && self.binlogServer.image + != '''')' + - message: binlogServer.size is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.size) && self.binlogServer.size + > 0)' resources: properties: claims: @@ -8039,10 +8047,12 @@ spec: type: string type: object type: object - required: - - image - - size type: object + x-kubernetes-validations: + - message: mysql.image is required + rule: has(self.image) && self.image != '' + - message: mysql.size must be greater than 0 + rule: has(self.size) && self.size > 0 orchestrator: properties: affinity: @@ -9283,10 +9293,15 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: orchestrator.image is required when orchestrator is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: orchestrator.size must be greater than 0 when orchestrator + is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && + self.size > 0)' pause: type: boolean pmm: @@ -9568,9 +9583,11 @@ spec: type: object serverHost: type: string - required: - - image type: object + x-kubernetes-validations: + - message: pmm.image is required when pmm is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' proxy: properties: haproxy: @@ -10813,10 +10830,15 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: haproxy.image is required when haproxy is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: haproxy.size must be greater than 0 when haproxy is + enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) + && self.size > 0)' router: properties: affinity: @@ -12082,10 +12104,14 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: router.image is required when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: router.size must be greater than 0 when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) + && self.size > 0)' type: object secretsName: type: string @@ -12580,9 +12606,10 @@ spec: format: int32 type: integer type: object - required: - - image type: object + x-kubernetes-validations: + - message: toolkit.image is required + rule: has(self.image) && self.image != '' unsafeFlags: properties: mysqlSize: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index e0962baf0..4ea465697 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -3853,16 +3853,24 @@ spec: writeTimeout: format: int32 type: integer - required: - - image - - size type: object x-kubernetes-validations: - message: binlogServer size cannot be more than 1 - rule: self.size <= 1 + rule: '!has(self.size) || self.size <= 1' enabled: type: boolean type: object + x-kubernetes-validations: + - message: binlogServer is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' + - message: binlogServer.image is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.image) && self.binlogServer.image + != '''')' + - message: binlogServer.size is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.size) && self.binlogServer.size + > 0)' resources: properties: claims: @@ -8039,10 +8047,12 @@ spec: type: string type: object type: object - required: - - image - - size type: object + x-kubernetes-validations: + - message: mysql.image is required + rule: has(self.image) && self.image != '' + - message: mysql.size must be greater than 0 + rule: has(self.size) && self.size > 0 orchestrator: properties: affinity: @@ -9283,10 +9293,15 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: orchestrator.image is required when orchestrator is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: orchestrator.size must be greater than 0 when orchestrator + is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && + self.size > 0)' pause: type: boolean pmm: @@ -9568,9 +9583,11 @@ spec: type: object serverHost: type: string - required: - - image type: object + x-kubernetes-validations: + - message: pmm.image is required when pmm is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' proxy: properties: haproxy: @@ -10813,10 +10830,15 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: haproxy.image is required when haproxy is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: haproxy.size must be greater than 0 when haproxy is + enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) + && self.size > 0)' router: properties: affinity: @@ -12082,10 +12104,14 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: router.image is required when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: router.size must be greater than 0 when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) + && self.size > 0)' type: object secretsName: type: string @@ -12580,9 +12606,10 @@ spec: format: int32 type: integer type: object - required: - - image type: object + x-kubernetes-validations: + - message: toolkit.image is required + rule: has(self.image) && self.image != '' unsafeFlags: properties: mysqlSize: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 660eb12d1..764918915 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -3853,16 +3853,24 @@ spec: writeTimeout: format: int32 type: integer - required: - - image - - size type: object x-kubernetes-validations: - message: binlogServer size cannot be more than 1 - rule: self.size <= 1 + rule: '!has(self.size) || self.size <= 1' enabled: type: boolean type: object + x-kubernetes-validations: + - message: binlogServer is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' + - message: binlogServer.image is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.image) && self.binlogServer.image + != '''')' + - message: binlogServer.size is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.size) && self.binlogServer.size + > 0)' resources: properties: claims: @@ -8039,10 +8047,12 @@ spec: type: string type: object type: object - required: - - image - - size type: object + x-kubernetes-validations: + - message: mysql.image is required + rule: has(self.image) && self.image != '' + - message: mysql.size must be greater than 0 + rule: has(self.size) && self.size > 0 orchestrator: properties: affinity: @@ -9283,10 +9293,15 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: orchestrator.image is required when orchestrator is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: orchestrator.size must be greater than 0 when orchestrator + is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && + self.size > 0)' pause: type: boolean pmm: @@ -9568,9 +9583,11 @@ spec: type: object serverHost: type: string - required: - - image type: object + x-kubernetes-validations: + - message: pmm.image is required when pmm is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' proxy: properties: haproxy: @@ -10813,10 +10830,15 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: haproxy.image is required when haproxy is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: haproxy.size must be greater than 0 when haproxy is + enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) + && self.size > 0)' router: properties: affinity: @@ -12082,10 +12104,14 @@ spec: - whenUnsatisfiable type: object type: array - required: - - image - - size type: object + x-kubernetes-validations: + - message: router.image is required when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' + - message: router.size must be greater than 0 when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.size) + && self.size > 0)' type: object secretsName: type: string @@ -12580,9 +12606,10 @@ spec: format: int32 type: integer type: object - required: - - image type: object + x-kubernetes-validations: + - message: toolkit.image is required + rule: has(self.image) && self.image != '' unsafeFlags: properties: mysqlSize: diff --git a/pkg/controller/ps/controller_test.go b/pkg/controller/ps/controller_test.go index 12cb89a84..b63a1070c 100644 --- a/pkg/controller/ps/controller_test.go +++ b/pkg/controller/ps/controller_test.go @@ -596,10 +596,8 @@ var _ = Describe("CR validations", Ordered, func() { It("should fail the creation of cr", func() { err := k8sClient.Create(ctx, cr) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("spec.mysql.size: Required value")) - Expect(err.Error()).To(ContainSubstring("spec.proxy.haproxy.size: Required value")) - Expect(err.Error()).To(ContainSubstring("spec.proxy.router.size")) - Expect(err.Error()).To(ContainSubstring("spec.orchestrator.size")) + Expect(err.Error()).To(ContainSubstring("mysql.image is required")) + Expect(err.Error()).To(ContainSubstring("mysql.size must be greater than 0")) }) }) When("group-replication cluster", Ordered, func() { @@ -617,6 +615,7 @@ var _ = Describe("CR validations", Ordered, func() { cr.Spec.Proxy.Router.Enabled = false cr.Spec.MySQL.Image = "mysql-image" + cr.Spec.Toolkit.Image = "toolkit-image" cr.Spec.Proxy.HAProxy.Image = "haproxy-image" cr.Spec.Orchestrator.Image = "orc-image" @@ -950,6 +949,173 @@ var _ = Describe("CR validations", Ordered, func() { Expect(createErr.Error()).To(ContainSubstring("Invalid configuration: MySQL Router and HAProxy can't be enabled at the same time")) }) }) + + When("component image/size is missing but component is disabled", Ordered, func() { + cr, err := readDefaultCR("cr-validations-comp-disabled", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.MySQL.ClusterType = psv1.ClusterTypeGR + cr.Spec.Proxy.HAProxy.Enabled = true + cr.Spec.Proxy.Router.Enabled = false + cr.Spec.Proxy.Router.Image = "" + cr.Spec.Proxy.Router.Size = 0 + cr.Spec.Orchestrator.Enabled = false + cr.Spec.Orchestrator.Image = "" + cr.Spec.Orchestrator.Size = 0 + It("should create the cluster successfully", func() { + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + }) + + When("haproxy is enabled but image is missing", Ordered, func() { + cr, err := readDefaultCR("cr-validations-haproxy-no-image", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Proxy.HAProxy.Enabled = true + cr.Spec.Proxy.HAProxy.Image = "" + It("should fail with image required error", func() { + createErr := k8sClient.Create(ctx, cr) + Expect(createErr).To(HaveOccurred()) + Expect(createErr.Error()).To(ContainSubstring("haproxy.image is required when haproxy is enabled")) + }) + }) + + When("haproxy is enabled but size is 0", Ordered, func() { + cr, err := readDefaultCR("cr-validations-haproxy-no-size", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Proxy.HAProxy.Enabled = true + cr.Spec.Proxy.HAProxy.Size = 0 + It("should fail with size required error", func() { + createErr := k8sClient.Create(ctx, cr) + Expect(createErr).To(HaveOccurred()) + Expect(createErr.Error()).To(ContainSubstring("haproxy.size must be greater than 0 when haproxy is enabled")) + }) + }) + + When("orchestrator is enabled but image is missing", Ordered, func() { + cr, err := readDefaultCR("cr-validations-orc-no-image", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.MySQL.ClusterType = psv1.ClusterTypeAsync + cr.Spec.Orchestrator.Enabled = true + cr.Spec.Orchestrator.Image = "" + It("should fail with image required error", func() { + createErr := k8sClient.Create(ctx, cr) + Expect(createErr).To(HaveOccurred()) + Expect(createErr.Error()).To(ContainSubstring("orchestrator.image is required when orchestrator is enabled")) + }) + }) + + When("router is enabled but image is missing", Ordered, func() { + cr, err := readDefaultCR("cr-validations-router-no-image", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.MySQL.ClusterType = psv1.ClusterTypeGR + cr.Spec.Proxy.Router.Enabled = true + cr.Spec.Proxy.Router.Image = "" + It("should fail with image required error", func() { + createErr := k8sClient.Create(ctx, cr) + Expect(createErr).To(HaveOccurred()) + Expect(createErr.Error()).To(ContainSubstring("router.image is required when router is enabled")) + }) + }) + }) + + Context("PITR validation rules", Ordered, func() { + ns := "validate-pitr" + + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + } + + BeforeAll(func() { + By("Creating the Namespace to perform the tests") + err := k8sClient.Create(ctx, namespace) + Expect(err).To(Not(HaveOccurred())) + }) + + AfterAll(func() { + By("Deleting the Namespace") + _ = k8sClient.Delete(ctx, namespace) + }) + + When("pitr is disabled, no binlogServer required", Ordered, func() { + cr, err := readDefaultCR("pitr-disabled-no-binlog", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Backup.PiTR.Enabled = false + It("should create successfully without any binlogServer fields", func() { + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + }) + + When("pitr is disabled, binlogServer provided without image and size", Ordered, func() { + cr, err := readDefaultCR("pitr-disabled-empty-binlog", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Backup.PiTR.Enabled = false + cr.Spec.Backup.PiTR.BinlogServer = &psv1.BinlogServerSpec{} + It("should create successfully since pitr is disabled", func() { + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + }) + + When("pitr is enabled but binlogServer is missing", Ordered, func() { + cr, err := readDefaultCR("pitr-enabled-no-binlog", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Backup.PiTR.Enabled = true + cr.Spec.Backup.PiTR.BinlogServer = nil + It("should fail with binlogServer required error", func() { + createErr := k8sClient.Create(ctx, cr) + Expect(createErr).To(HaveOccurred()) + Expect(createErr.Error()).To(ContainSubstring("binlogServer is required when pitr is enabled")) + }) + }) + + When("pitr is enabled but binlogServer image is missing", Ordered, func() { + cr, err := readDefaultCR("pitr-enabled-no-image", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Backup.PiTR.Enabled = true + cr.Spec.Backup.PiTR.BinlogServer = &psv1.BinlogServerSpec{} + cr.Spec.Backup.PiTR.BinlogServer.Size = 1 + It("should fail with image required error", func() { + createErr := k8sClient.Create(ctx, cr) + Expect(createErr).To(HaveOccurred()) + Expect(createErr.Error()).To(ContainSubstring("binlogServer.image is required when pitr is enabled")) + }) + }) + + When("pitr is enabled but binlogServer size is 0", Ordered, func() { + cr, err := readDefaultCR("pitr-enabled-no-size", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Backup.PiTR.Enabled = true + cr.Spec.Backup.PiTR.BinlogServer = &psv1.BinlogServerSpec{} + cr.Spec.Backup.PiTR.BinlogServer.Image = "binlog-server-image" + It("should fail with size required error", func() { + createErr := k8sClient.Create(ctx, cr) + Expect(createErr).To(HaveOccurred()) + Expect(createErr.Error()).To(ContainSubstring("binlogServer.size is required when pitr is enabled")) + }) + }) + + When("pitr is enabled with all required fields set", Ordered, func() { + cr, err := readDefaultCR("pitr-enabled-valid", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Backup.PiTR.Enabled = true + cr.Spec.Backup.PiTR.BinlogServer = &psv1.BinlogServerSpec{} + cr.Spec.Backup.PiTR.BinlogServer.Image = "binlog-server-image" + cr.Spec.Backup.PiTR.BinlogServer.Size = 1 + It("should create successfully", func() { + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + }) }) }) From 3b866839d961c9a5fdf9baa10ed0a58cff031088 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 11:31:24 +0300 Subject: [PATCH 048/102] add unit tests for pitr restore job and related functions --- pkg/pitr/pitr.go | 12 +- pkg/pitr/pitr_test.go | 604 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 610 insertions(+), 6 deletions(-) create mode 100644 pkg/pitr/pitr_test.go diff --git a/pkg/pitr/pitr.go b/pkg/pitr/pitr.go index a7e2e0b7e..13fe1a4fc 100644 --- a/pkg/pitr/pitr.go +++ b/pkg/pitr/pitr.go @@ -33,10 +33,6 @@ func JobName(restore *apiv1.PerconaServerMySQLRestore) string { return fmt.Sprintf("pitr-restore-%s", restore.Name) } -func BinlogsConfigMapName(restore *apiv1.PerconaServerMySQLRestore) string { - return fmt.Sprintf("pitr-binlogs-%s", restore.Name) -} - func BinlogsConfigMap(cluster *apiv1.PerconaServerMySQL, restore *apiv1.PerconaServerMySQLRestore) *corev1.ConfigMap { labels := util.SSMapMerge(cluster.GlobalLabels(), restore.Labels(appName, naming.ComponentPITR)) @@ -46,7 +42,7 @@ func BinlogsConfigMap(cluster *apiv1.PerconaServerMySQL, restore *apiv1.PerconaS Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ - Name: BinlogsConfigMapName(restore), + Name: binlogsConfigMapName(restore), Namespace: cluster.Namespace, Labels: labels, Annotations: cluster.GlobalAnnotations(), @@ -159,7 +155,7 @@ func RestoreJob( VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: BinlogsConfigMapName(restore), + Name: binlogsConfigMapName(restore), }, }, }, @@ -280,3 +276,7 @@ func restoreContainer( Resources: storage.Resources, } } + +func binlogsConfigMapName(restore *apiv1.PerconaServerMySQLRestore) string { + return fmt.Sprintf("pitr-binlogs-%s", restore.Name) +} diff --git a/pkg/pitr/pitr_test.go b/pkg/pitr/pitr_test.go new file mode 100644 index 000000000..a402e8eab --- /dev/null +++ b/pkg/pitr/pitr_test.go @@ -0,0 +1,604 @@ +package pitr + +import ( + "testing" + + "github.com/stretchr/testify/assert" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/mysql" +) + +func TestRestoreJob(t *testing.T) { + tests := map[string]struct { + cluster *apiv1.PerconaServerMySQL + restore *apiv1.PerconaServerMySQLRestore + storage *apiv1.BackupStorageSpec + initImage string + verify func(t *testing.T, job *batchv1.Job) + }{ + "basic job metadata": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "test-ns", + }, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "my-cluster-secrets", + SSLSecretName: "my-cluster-ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-restore", + Namespace: "test-ns", + }, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + assert.Equal(t, "pitr-restore-my-restore", job.Name) + assert.Equal(t, "test-ns", job.Namespace) + assert.Equal(t, "batch/v1", job.TypeMeta.APIVersion) + assert.Equal(t, "Job", job.TypeMeta.Kind) + }, + }, + "job spec parallelism and completions": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "cluster-secrets", + SSLSecretName: "cluster-ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + assert.Equal(t, ptr.To(int32(1)), job.Spec.Parallelism) + assert.Equal(t, ptr.To(int32(1)), job.Spec.Completions) + assert.Equal(t, corev1.RestartPolicyNever, job.Spec.Template.Spec.RestartPolicy) + }, + }, + "backoff limit from cluster spec": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "cluster-secrets", + SSLSecretName: "cluster-ssl", + Backup: &apiv1.BackupSpec{ + BackoffLimit: ptr.To(int32(5)), + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + assert.Equal(t, ptr.To(int32(5)), job.Spec.BackoffLimit) + }, + }, + "pvc name uses cluster name": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "mycluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "mycluster-secrets", + SSLSecretName: "mycluster-ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + volumes := job.Spec.Template.Spec.Volumes + expectedPVCName := mysql.DataVolumeName + "-mycluster-mysql-0" + var found bool + for _, v := range volumes { + if v.Name == dataVolumeName && v.PersistentVolumeClaim != nil { + assert.Equal(t, expectedPVCName, v.PersistentVolumeClaim.ClaimName) + found = true + } + } + assert.True(t, found, "datadir volume with expected PVC not found") + }, + }, + "volumes include all expected volumes": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "my-secrets", + SSLSecretName: "my-ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + volumeNames := map[string]bool{} + for _, v := range job.Spec.Template.Spec.Volumes { + volumeNames[v.Name] = true + } + assert.True(t, volumeNames[apiv1.BinVolumeName], "missing bin volume") + assert.True(t, volumeNames[dataVolumeName], "missing datadir volume") + assert.True(t, volumeNames[credsVolumeName], "missing creds volume") + assert.True(t, volumeNames[tlsVolumeName], "missing tls volume") + assert.True(t, volumeNames[binlogsVolumeName], "missing binlogs volume") + }, + }, + "secrets volume uses cluster secrets name": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "custom-secrets", + SSLSecretName: "custom-ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + for _, v := range job.Spec.Template.Spec.Volumes { + switch v.Name { + case credsVolumeName: + assert.Equal(t, "custom-secrets", v.Secret.SecretName) + case tlsVolumeName: + assert.Equal(t, "custom-ssl", v.Secret.SecretName) + } + } + }, + }, + "binlogs configmap volume references restore name": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + for _, v := range job.Spec.Template.Spec.Volumes { + if v.Name == binlogsVolumeName { + assert.Equal(t, "pitr-binlogs-my-restore", v.ConfigMap.Name) + return + } + } + t.Error("binlogs volume not found") + }, + }, + "storage scheduling fields propagated to pod spec": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{ + NodeSelector: map[string]string{"disktype": "ssd"}, + SchedulerName: "my-scheduler", + PriorityClassName: "high-priority", + Tolerations: []corev1.Toleration{ + {Key: "dedicated", Operator: corev1.TolerationOpEqual, Value: "mysql", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + spec := job.Spec.Template.Spec + assert.Equal(t, map[string]string{"disktype": "ssd"}, spec.NodeSelector) + assert.Equal(t, "my-scheduler", spec.SchedulerName) + assert.Equal(t, "high-priority", spec.PriorityClassName) + assert.Len(t, spec.Tolerations, 1) + assert.Equal(t, "dedicated", spec.Tolerations[0].Key) + }, + }, + "image pull secrets from backup spec": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "registry-secret"}}, + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + assert.Equal(t, []corev1.LocalObjectReference{{Name: "registry-secret"}}, job.Spec.Template.Spec.ImagePullSecrets) + }, + }, + "restore container has correct env vars without pitr spec": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLRestoreSpec{}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + container := job.Spec.Template.Spec.Containers[0] + envMap := envToMap(container.Env) + assert.Equal(t, "my-restore", envMap["RESTORE_NAME"]) + assert.Equal(t, binlogsMountPath+"/"+BinlogsConfigKey, envMap["BINLOGS_PATH"]) + assert.NotContains(t, envMap, "PITR_TYPE") + assert.NotContains(t, envMap, "PITR_DATE") + assert.NotContains(t, envMap, "PITR_GTID") + }, + }, + "restore container has pitr date env vars": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLRestoreSpec{ + PITR: &apiv1.RestorePITRSpec{ + Type: apiv1.PITRDate, + Date: "2024-01-15 10:00:00", + }, + }, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + container := job.Spec.Template.Spec.Containers[0] + envMap := envToMap(container.Env) + assert.Equal(t, "date", envMap["PITR_TYPE"]) + assert.Equal(t, "2024-01-15 10:00:00", envMap["PITR_DATE"]) + assert.NotContains(t, envMap, "PITR_GTID") + }, + }, + "restore container has pitr gtid env vars": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLRestoreSpec{ + PITR: &apiv1.RestorePITRSpec{ + Type: apiv1.PITRGtid, + GTID: "abc123:1-100", + }, + }, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + container := job.Spec.Template.Spec.Containers[0] + envMap := envToMap(container.Env) + assert.Equal(t, "gtid", envMap["PITR_TYPE"]) + assert.Equal(t, "abc123:1-100", envMap["PITR_GTID"]) + assert.NotContains(t, envMap, "PITR_DATE") + }, + }, + "restore container has s3 env vars when binlog server has s3 storage": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{ + Storage: apiv1.BinlogServerStorageSpec{ + S3: &apiv1.BackupStorageS3Spec{ + Bucket: "my-bucket", + CredentialsSecret: "s3-creds", + Region: "us-east-1", + EndpointURL: "https://s3.example.com", + }, + }, + }, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + container := job.Spec.Template.Spec.Containers[0] + envMap := envToMap(container.Env) + assert.Equal(t, "s3", envMap["STORAGE_TYPE"]) + assert.Equal(t, "my-bucket", envMap["S3_BUCKET"]) + assert.Equal(t, "us-east-1", envMap["AWS_DEFAULT_REGION"]) + assert.Equal(t, "https://s3.example.com", envMap["AWS_ENDPOINT"]) + + envByName := envByNameMap(container.Env) + accessKey := envByName["AWS_ACCESS_KEY_ID"] + assert.NotNil(t, accessKey.ValueFrom) + assert.Equal(t, "s3-creds", accessKey.ValueFrom.SecretKeyRef.Name) + assert.Equal(t, "AWS_ACCESS_KEY_ID", accessKey.ValueFrom.SecretKeyRef.Key) + + secretKey := envByName["AWS_SECRET_ACCESS_KEY"] + assert.NotNil(t, secretKey.ValueFrom) + assert.Equal(t, "s3-creds", secretKey.ValueFrom.SecretKeyRef.Name) + assert.Equal(t, "AWS_SECRET_ACCESS_KEY", secretKey.ValueFrom.SecretKeyRef.Key) + }, + }, + "restore container command and volume mounts": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + MySQL: apiv1.MySQLSpec{}, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + assert.Len(t, job.Spec.Template.Spec.Containers, 1) + container := job.Spec.Template.Spec.Containers[0] + assert.Equal(t, appName, container.Name) + assert.Equal(t, []string{"/opt/percona/run-pitr-restore.sh"}, container.Command) + + mountNames := map[string]bool{} + for _, m := range container.VolumeMounts { + mountNames[m.Name] = true + } + assert.True(t, mountNames[apiv1.BinVolumeName]) + assert.True(t, mountNames[dataVolumeName]) + assert.True(t, mountNames[credsVolumeName]) + assert.True(t, mountNames[tlsVolumeName]) + assert.True(t, mountNames[binlogsVolumeName]) + }, + }, + "one init container present": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "percona/init:1.0", + verify: func(t *testing.T, job *batchv1.Job) { + assert.Len(t, job.Spec.Template.Spec.InitContainers, 1) + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + job := RestoreJob(tt.cluster, tt.restore, tt.storage, tt.initImage) + tt.verify(t, job) + }) + } +} + +func envToMap(envs []corev1.EnvVar) map[string]string { + m := make(map[string]string, len(envs)) + for _, e := range envs { + m[e.Name] = e.Value + } + return m +} + +func envByNameMap(envs []corev1.EnvVar) map[string]corev1.EnvVar { + m := make(map[string]corev1.EnvVar, len(envs)) + for _, e := range envs { + m[e.Name] = e + } + return m +} + +func TestJobName(t *testing.T) { + tests := map[string]struct { + restoreName string + expected string + }{ + "simple name": {restoreName: "my-restore", expected: "pitr-restore-my-restore"}, + "name with numbers": {restoreName: "restore-123", expected: "pitr-restore-restore-123"}, + "single word name": {restoreName: "restore", expected: "pitr-restore-restore"}, + "name with dots": {restoreName: "restore.v1", expected: "pitr-restore-restore.v1"}, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + restore := &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: tt.restoreName}, + } + assert.Equal(t, tt.expected, JobName(restore)) + }) + } +} + +func TestBinlogsConfigMap(t *testing.T) { + tests := map[string]struct { + cluster *apiv1.PerconaServerMySQL + restore *apiv1.PerconaServerMySQLRestore + verify func(t *testing.T, cm *corev1.ConfigMap) + }{ + "basic metadata": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "my-cluster", Namespace: "test-ns"}, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore"}, + }, + verify: func(t *testing.T, cm *corev1.ConfigMap) { + assert.Equal(t, "pitr-binlogs-my-restore", cm.Name) + assert.Equal(t, "test-ns", cm.Namespace) + assert.Equal(t, "v1", cm.TypeMeta.APIVersion) + assert.Equal(t, "ConfigMap", cm.TypeMeta.Kind) + }, + }, + "no global labels or annotations": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore"}, + }, + verify: func(t *testing.T, cm *corev1.ConfigMap) { + assert.Nil(t, cm.Annotations) + }, + }, + "global labels merged into configmap labels": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + Metadata: &apiv1.Metadata{ + Labels: map[string]string{"env": "prod"}, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore"}, + }, + verify: func(t *testing.T, cm *corev1.ConfigMap) { + assert.Equal(t, "prod", cm.Labels["env"]) + }, + }, + "global annotations propagated": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + Metadata: &apiv1.Metadata{ + Annotations: map[string]string{"team": "dba"}, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "restore"}, + }, + verify: func(t *testing.T, cm *corev1.ConfigMap) { + assert.Equal(t, "dba", cm.Annotations["team"]) + }, + }, + "name derived from restore name": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "weekly-restore"}, + }, + verify: func(t *testing.T, cm *corev1.ConfigMap) { + assert.Equal(t, "pitr-binlogs-weekly-restore", cm.Name) + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cm := BinlogsConfigMap(tt.cluster, tt.restore) + tt.verify(t, cm) + }) + } +} From 4d844a174182ba92ba128d4a59cc5b66c6cf73f0 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 11:44:31 +0300 Subject: [PATCH 049/102] unit test sts of bls --- pkg/binlogserver/binlog_server_test.go | 310 +++++++++++++++++++++++++ pkg/binlogserver/config.go | 3 +- 2 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 pkg/binlogserver/binlog_server_test.go diff --git a/pkg/binlogserver/binlog_server_test.go b/pkg/binlogserver/binlog_server_test.go new file mode 100644 index 000000000..37a832fd3 --- /dev/null +++ b/pkg/binlogserver/binlog_server_test.go @@ -0,0 +1,310 @@ +package binlogserver + +import ( + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/naming" +) + +func newTestCR(name, namespace string) *apiv1.PerconaServerMySQL { + return &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: apiv1.PerconaServerMySQLSpec{ + SSLSecretName: name + "-ssl", + SecretsName: name + "-secrets", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{ + Storage: apiv1.BinlogServerStorageSpec{ + S3: &apiv1.BackupStorageS3Spec{ + CredentialsSecret: "s3-creds-secret", + }, + }, + }, + }, + }, + }, + } +} + +func TestStatefulSet(t *testing.T) { + tests := map[string]struct { + cr *apiv1.PerconaServerMySQL + initImage string + configHash string + verify func(t *testing.T, cr *apiv1.PerconaServerMySQL) + }{ + "object meta": { + cr: newTestCR("my-cluster", "test-ns"), + initImage: "init:latest", + configHash: "abc123", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "abc123") + + assert.Equal(t, "apps/v1", sts.TypeMeta.APIVersion) + assert.Equal(t, "StatefulSet", sts.TypeMeta.Kind) + assert.Equal(t, "my-cluster-binlog-server", sts.Name) + assert.Equal(t, "test-ns", sts.Namespace) + }, + }, + "labels": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + + expectedLabels := MatchLabels(cr) + assert.Equal(t, expectedLabels, sts.Labels) + assert.Equal(t, expectedLabels, sts.Spec.Selector.MatchLabels) + assert.Equal(t, expectedLabels, sts.Spec.Template.Labels) + }, + }, + "replicas is always 1": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + assert.Equal(t, ptr.To(int32(1)), sts.Spec.Replicas) + }, + }, + "config hash annotation in pod template": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + configHash: "deadbeef", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "2pacisnotdead") + assert.Equal(t, "deadbeef", sts.Spec.Template.Annotations[string(naming.AnnotationConfigHash)]) + }, + }, + "empty config hash produces no annotation": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + assert.NotContains(t, sts.Spec.Template.Annotations, string(naming.AnnotationConfigHash)) + }, + }, + "init container uses provided image": { + cr: newTestCR("cluster", "ns"), + initImage: "percona/init:1.2.3", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "percona/init:1.2.3", "") + assert.Len(t, sts.Spec.Template.Spec.InitContainers, 1) + assert.Equal(t, "percona/init:1.2.3", sts.Spec.Template.Spec.InitContainers[0].Image) + }, + }, + "binlog server container present with correct name": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + assert.Len(t, sts.Spec.Template.Spec.Containers, 1) + assert.Equal(t, AppName, sts.Spec.Template.Spec.Containers[0].Name) + }, + }, + "container command and args": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + container := sts.Spec.Template.Spec.Containers[0] + assert.Equal(t, []string{"/opt/percona/binlog-server-entrypoint.sh"}, container.Command) + assert.Equal(t, []string{ + binlogServerBinary, + "pull", + configMountPath + "/" + ConfigKey, + }, container.Args) + }, + }, + "binlog server container image from spec": { + cr: func() *apiv1.PerconaServerMySQL { + cr := newTestCR("cluster", "ns") + cr.Spec.Backup.PiTR.BinlogServer.Image = "percona/binlog-server:2.0" + return cr + }(), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + container := sts.Spec.Template.Spec.Containers[0] + assert.Equal(t, "percona/binlog-server:2.0", container.Image) + }, + }, + "volumes include all expected volumes": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + volumeNames := map[string]bool{} + for _, v := range sts.Spec.Template.Spec.Volumes { + volumeNames[v.Name] = true + } + assert.True(t, volumeNames[apiv1.BinVolumeName], "missing bin volume") + assert.True(t, volumeNames[bufferVolumeName], "missing buffer volume") + assert.True(t, volumeNames[credsVolumeName], "missing creds volume") + assert.True(t, volumeNames[tlsVolumeName], "missing tls volume") + assert.True(t, volumeNames[storageCredsVolumeName], "missing storage volume") + assert.True(t, volumeNames[configVolumeName], "missing config volume") + }, + }, + "creds volume uses internal secret name": { + cr: newTestCR("mycluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.Name == credsVolumeName { + assert.Equal(t, cr.InternalSecretName(), v.Secret.SecretName) + return + } + } + t.Error("creds volume not found") + }, + }, + "tls volume uses ssl secret name": { + cr: newTestCR("mycluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.Name == tlsVolumeName { + assert.Equal(t, "mycluster-ssl", v.Secret.SecretName) + return + } + } + t.Error("tls volume not found") + }, + }, + "storage volume uses s3 credentials secret": { + cr: func() *apiv1.PerconaServerMySQL { + cr := newTestCR("cluster", "ns") + cr.Spec.Backup.PiTR.BinlogServer.Storage.S3.CredentialsSecret = "my-s3-creds" + return cr + }(), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.Name == storageCredsVolumeName { + assert.Equal(t, "my-s3-creds", v.Secret.SecretName) + return + } + } + t.Error("storage volume not found") + }, + }, + "config volume references config secret": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.Name == configVolumeName { + assert.NotNil(t, v.Projected) + var hasConfigSecret bool + for _, src := range v.Projected.Sources { + if src.Secret != nil && src.Secret.Name == ConfigSecretName(cr) { + hasConfigSecret = true + } + } + assert.True(t, hasConfigSecret, "config volume should reference config secret") + return + } + } + t.Error("config volume not found") + }, + }, + "global annotations propagated to statefulset": { + cr: func() *apiv1.PerconaServerMySQL { + cr := newTestCR("cluster", "ns") + cr.Spec.Metadata = &apiv1.Metadata{ + Annotations: map[string]string{"team": "dba"}, + } + return cr + }(), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + assert.Equal(t, "dba", sts.Annotations["team"]) + }, + }, + "global labels propagated to statefulset": { + cr: func() *apiv1.PerconaServerMySQL { + cr := newTestCR("cluster", "ns") + cr.Spec.Metadata = &apiv1.Metadata{ + Labels: map[string]string{"env": "prod"}, + } + return cr + }(), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + assert.Equal(t, "prod", sts.Labels["env"]) + }, + }, + "container volume mounts include all expected mounts": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + container := sts.Spec.Template.Spec.Containers[0] + mountNames := map[string]bool{} + for _, m := range container.VolumeMounts { + mountNames[m.Name] = true + } + assert.True(t, mountNames[apiv1.BinVolumeName], "missing bin volume mount") + assert.True(t, mountNames[credsVolumeName], "missing creds volume mount") + assert.True(t, mountNames[tlsVolumeName], "missing tls volume mount") + assert.True(t, mountNames[configVolumeName], "missing config volume mount") + assert.True(t, mountNames[bufferVolumeName], "missing buffer volume mount") + }, + }, + "container env includes CONFIG_PATH and CUSTOM_CONFIG_PATH": { + cr: newTestCR("cluster", "ns"), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + container := sts.Spec.Template.Spec.Containers[0] + envMap := make(map[string]string) + for _, e := range container.Env { + envMap[e.Name] = e.Value + } + assert.Contains(t, envMap, "CONFIG_PATH") + assert.Contains(t, envMap, "CUSTOM_CONFIG_PATH") + }, + }, + "custom env vars from spec are appended": { + cr: func() *apiv1.PerconaServerMySQL { + cr := newTestCR("cluster", "ns") + cr.Spec.Backup.PiTR.BinlogServer.Env = []corev1.EnvVar{ + {Name: "MY_CUSTOM_VAR", Value: "custom-value"}, + } + return cr + }(), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + container := sts.Spec.Template.Spec.Containers[0] + envMap := make(map[string]string) + for _, e := range container.Env { + envMap[e.Name] = e.Value + } + assert.Equal(t, "custom-value", envMap["MY_CUSTOM_VAR"]) + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + tt.verify(t, tt.cr) + }) + } +} diff --git a/pkg/binlogserver/config.go b/pkg/binlogserver/config.go index a95218576..f0cb65739 100644 --- a/pkg/binlogserver/config.go +++ b/pkg/binlogserver/config.go @@ -43,8 +43,7 @@ type Connection struct { type ReplicationMode string const ( - ReplicationModeGTID ReplicationMode = "gtid" - ReplicationModePosition ReplicationMode = "position" + ReplicationModeGTID ReplicationMode = "gtid" ) type Rewrite struct { From 9a037228bdfc8096bb2ec2b337098bcd346a0206 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 12:01:53 +0300 Subject: [PATCH 050/102] unit test for gtid and date search --- pkg/binlogserver/search_test.go | 235 ++++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 pkg/binlogserver/search_test.go diff --git a/pkg/binlogserver/search_test.go b/pkg/binlogserver/search_test.go new file mode 100644 index 000000000..2311e0f99 --- /dev/null +++ b/pkg/binlogserver/search_test.go @@ -0,0 +1,235 @@ +package binlogserver + +import ( + "context" + "encoding/json" + "fmt" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + restclient "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/pkg/clientcmd" +) + +type fakeExecClient struct { + response *SearchResponse + execErr error + capturedCmd []string +} + +var _ clientcmd.Client = (*fakeExecClient)(nil) + +func (f *fakeExecClient) Exec(_ context.Context, _ *corev1.Pod, _ string, cmd []string, _ io.Reader, stdout, _ io.Writer, _ bool) error { + f.capturedCmd = cmd + if f.execErr != nil { + return f.execErr + } + if stdout != nil && f.response != nil { + data, _ := json.Marshal(f.response) + stdout.Write(data) + } + return nil +} + +func (f *fakeExecClient) REST() restclient.Interface { + return nil +} + +func newReadyBinlogServerPod(cr *apiv1.PerconaServerMySQL) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: Name(cr) + "-0", + Namespace: cr.Namespace, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + { + Type: corev1.ContainersReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } +} + +func newSearchTestClient(t *testing.T, pod *corev1.Pod) *fake.ClientBuilder { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, apiv1.AddToScheme(scheme)) + cb := fake.NewClientBuilder().WithScheme(scheme) + if pod != nil { + cb = cb.WithObjects(pod) + } + return cb +} + +func TestSearchByGTID(t *testing.T) { + cr := newTestCR("my-cluster", "test-ns") + + successResponse := &SearchResponse{ + Version: 1, + Status: "OK", + Result: []BinlogEntry{ + { + Name: "binlog.000001", + PreviousGTIDs: "00000000-0000-0000-0000-000000000000:1-10", + AddedGTIDs: "00000000-0000-0000-0000-000000000000:11", + }, + }, + } + + tests := map[string]struct { + pod *corev1.Pod + cliCmd clientcmd.Client + gtidSet string + expectedResponse *SearchResponse + expectedError string + }{ + "success": { + pod: newReadyBinlogServerPod(cr), + cliCmd: &fakeExecClient{response: successResponse}, + gtidSet: "00000000-0000-0000-0000-000000000000:1-10", + expectedResponse: successResponse, + }, + "pod not found": { + cliCmd: &fakeExecClient{}, + gtidSet: "some-gtid", + expectedError: "get binlog server pod", + }, + "pod not ready": { + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: Name(cr) + "-0", + Namespace: cr.Namespace, + }, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + }, + cliCmd: &fakeExecClient{}, + gtidSet: "some-gtid", + expectedError: "is not ready", + }, + "exec error": { + pod: newReadyBinlogServerPod(cr), + cliCmd: &fakeExecClient{execErr: fmt.Errorf("exec failed")}, + gtidSet: "some-gtid", + expectedError: "exec binlog_server search_by_gtid_set", + }, + "invalid json response": { + pod: newReadyBinlogServerPod(cr), + cliCmd: &fakeExecClient{response: nil}, + gtidSet: "some-gtid", + expectedError: "unmarshal response", + }, + } + + configPath := configMountPath + "/" + ConfigKey + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cl := newSearchTestClient(t, tt.pod).Build() + + execClient, _ := tt.cliCmd.(*fakeExecClient) + resp, err := SearchByGTID(t.Context(), cl, tt.cliCmd, cr, tt.gtidSet) + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, resp) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expectedResponse, resp) + assert.Equal(t, []string{binlogServerBinary, "search_by_gtid_set", configPath, tt.gtidSet}, execClient.capturedCmd) + }) + } +} + +func TestSearchByTimestamp(t *testing.T) { + cr := newTestCR("my-cluster", "test-ns") + + successResponse := &SearchResponse{ + Version: 1, + Status: "OK", + Result: []BinlogEntry{ + { + Name: "binlog.000002", + MinTimestamp: "2024-01-01 00:00:00", + MaxTimestamp: "2024-01-01 01:00:00", + }, + }, + } + + tests := map[string]struct { + pod *corev1.Pod + cliCmd clientcmd.Client + timestamp string + expectedResponse *SearchResponse + expectedError string + }{ + "success": { + pod: newReadyBinlogServerPod(cr), + cliCmd: &fakeExecClient{response: successResponse}, + timestamp: "2024-01-01 00:30:00", + expectedResponse: successResponse, + }, + "pod not found": { + cliCmd: &fakeExecClient{}, + timestamp: "2024-01-01 00:30:00", + expectedError: "get binlog server pod", + }, + "pod not ready": { + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: Name(cr) + "-0", + Namespace: cr.Namespace, + }, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + }, + cliCmd: &fakeExecClient{}, + timestamp: "2024-01-01 00:30:00", + expectedError: "is not ready", + }, + "exec error": { + pod: newReadyBinlogServerPod(cr), + cliCmd: &fakeExecClient{execErr: fmt.Errorf("exec failed")}, + timestamp: "2024-01-01 00:30:00", + expectedError: "exec binlog_server search_by_timestamp", + }, + "invalid json response": { + pod: newReadyBinlogServerPod(cr), + cliCmd: &fakeExecClient{response: nil}, + timestamp: "2024-01-01 00:30:00", + expectedError: "unmarshal response", + }, + } + + configPath := configMountPath + "/" + ConfigKey + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cl := newSearchTestClient(t, tt.pod).Build() + + execClient, _ := tt.cliCmd.(*fakeExecClient) + resp, err := SearchByTimestamp(t.Context(), cl, tt.cliCmd, cr, tt.timestamp) + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, resp) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expectedResponse, resp) + assert.Equal(t, []string{binlogServerBinary, "search_by_timestamp", configPath, tt.timestamp}, execClient.capturedCmd) + }) + } +} From 9d7ae2eaab82f1d3a17043bf6093607baaf92d23 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 12:04:31 +0300 Subject: [PATCH 051/102] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- pkg/tls/tls_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/tls/tls_test.go b/pkg/tls/tls_test.go index 090843a96..caa6ed6a8 100644 --- a/pkg/tls/tls_test.go +++ b/pkg/tls/tls_test.go @@ -22,12 +22,12 @@ func TestDNSNames(t *testing.T) { }, }, expected: map[string]struct{}{ - "*.cluster1-mysql": {}, - "*.cluster1-mysql.default": {}, - "*.cluster1-mysql.default.svc": {}, - "cluster1-mysql-primary": {}, - "cluster1-mysql-primary.default": {}, - "cluster1-mysql-primary.default.svc": {}, + "*.cluster1-mysql": {}, + "*.cluster1-mysql.default": {}, + "*.cluster1-mysql.default.svc": {}, + "cluster1-mysql-primary": {}, + "cluster1-mysql-primary.default": {}, + "cluster1-mysql-primary.default.svc": {}, "*.cluster1-orchestrator": {}, "*.cluster1-orchestrator.default": {}, "*.cluster1-orchestrator.default.svc": {}, @@ -49,12 +49,12 @@ func TestDNSNames(t *testing.T) { }, }, expected: map[string]struct{}{ - "*.cluster1-mysql": {}, - "*.cluster1-mysql.default": {}, - "*.cluster1-mysql.default.svc": {}, - "cluster1-mysql-primary": {}, - "cluster1-mysql-primary.default": {}, - "cluster1-mysql-primary.default.svc": {}, + "*.cluster1-mysql": {}, + "*.cluster1-mysql.default": {}, + "*.cluster1-mysql.default.svc": {}, + "cluster1-mysql-primary": {}, + "cluster1-mysql-primary.default": {}, + "cluster1-mysql-primary.default.svc": {}, "*.cluster1-orchestrator": {}, "*.cluster1-orchestrator.default": {}, "*.cluster1-orchestrator.default.svc": {}, From 68d3859c699774f22ca52f17f7a63ae76fb43836 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 12:15:47 +0300 Subject: [PATCH 052/102] rename pitr-minio to gr-pitr-minio --- e2e-tests/run-distro.csv | 2 +- e2e-tests/run-pr.csv | 2 +- e2e-tests/run-release.csv | 2 +- .../tests/{pitr-minio => gr-pitr-minio}/00-assert.yaml | 0 .../{pitr-minio => gr-pitr-minio}/00-minio-secret.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/01-assert.yaml | 0 .../{pitr-minio => gr-pitr-minio}/01-deploy-operator.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/02-assert.yaml | 8 ++++---- .../{pitr-minio => gr-pitr-minio}/02-create-cluster.yaml | 0 .../{pitr-minio => gr-pitr-minio}/03-write-data.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/04-assert.yaml | 2 +- .../{pitr-minio => gr-pitr-minio}/04-create-backup.yaml | 4 ++-- .../{pitr-minio => gr-pitr-minio}/05-write-more-data.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/06-assert.yaml | 4 ++-- .../06-create-pitr-restore.yaml | 4 ++-- .../tests/{pitr-minio => gr-pitr-minio}/07-assert.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/07-read-data.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/08-assert.yaml | 4 ++-- .../08-create-date-restore.yaml | 4 ++-- .../tests/{pitr-minio => gr-pitr-minio}/09-assert.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/09-read-data.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/10-failover.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/11-assert.yaml | 4 ++-- .../11-write-and-backup.yaml | 2 +- .../tests/{pitr-minio => gr-pitr-minio}/12-assert.yaml | 4 ++-- .../12-write-and-restore.yaml | 4 ++-- .../tests/{pitr-minio => gr-pitr-minio}/13-assert.yaml | 0 .../tests/{pitr-minio => gr-pitr-minio}/13-read-data.yaml | 0 .../{pitr-minio => gr-pitr-minio}/14-switchover.yaml | 0 .../{pitr-minio => gr-pitr-minio}/98-drop-finalizer.yaml | 2 +- .../99-remove-cluster-gracefully.yaml | 2 +- 31 files changed, 27 insertions(+), 27 deletions(-) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/00-assert.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/00-minio-secret.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/01-assert.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/01-deploy-operator.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/02-assert.yaml (87%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/02-create-cluster.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/03-write-data.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/04-assert.yaml (87%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/04-create-backup.yaml (71%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/05-write-more-data.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/06-assert.yaml (88%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/06-create-pitr-restore.yaml (90%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/07-assert.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/07-read-data.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/08-assert.yaml (87%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/08-create-date-restore.yaml (82%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/09-assert.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/09-read-data.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/10-failover.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/11-assert.yaml (88%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/11-write-and-backup.yaml (91%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/12-assert.yaml (86%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/12-write-and-restore.yaml (85%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/13-assert.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/13-read-data.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/14-switchover.yaml (100%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/98-drop-finalizer.yaml (78%) rename e2e-tests/tests/{pitr-minio => gr-pitr-minio}/99-remove-cluster-gracefully.yaml (91%) diff --git a/e2e-tests/run-distro.csv b/e2e-tests/run-distro.csv index 440325046..ea39a814e 100644 --- a/e2e-tests/run-distro.csv +++ b/e2e-tests/run-distro.csv @@ -3,7 +3,7 @@ auto-config config config-router demand-backup-minio -pitr-minio +gr-pitr-minio demand-backup-cloud demand-backup-retry gr-demand-backup-minio diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index cd57880f9..8dc7f9220 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -8,7 +8,7 @@ config-router,8.0 config-router,8.4 demand-backup-minio,8.0 demand-backup-minio,8.4 -pitr-minio,8.4 +gr-pitr-minio,8.4 demand-backup-cloud,8.4 demand-backup-retry,8.4 async-data-at-rest-encryption,8.0 diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index e899192c3..ef4b3d44d 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -5,7 +5,7 @@ auto-config config config-router demand-backup-minio -pitr-minio +gr-pitr-minio demand-backup-cloud demand-backup-retry async-data-at-rest-encryption diff --git a/e2e-tests/tests/pitr-minio/00-assert.yaml b/e2e-tests/tests/gr-pitr-minio/00-assert.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/00-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/00-assert.yaml diff --git a/e2e-tests/tests/pitr-minio/00-minio-secret.yaml b/e2e-tests/tests/gr-pitr-minio/00-minio-secret.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/00-minio-secret.yaml rename to e2e-tests/tests/gr-pitr-minio/00-minio-secret.yaml diff --git a/e2e-tests/tests/pitr-minio/01-assert.yaml b/e2e-tests/tests/gr-pitr-minio/01-assert.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/01-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/01-assert.yaml diff --git a/e2e-tests/tests/pitr-minio/01-deploy-operator.yaml b/e2e-tests/tests/gr-pitr-minio/01-deploy-operator.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/01-deploy-operator.yaml rename to e2e-tests/tests/gr-pitr-minio/01-deploy-operator.yaml diff --git a/e2e-tests/tests/pitr-minio/02-assert.yaml b/e2e-tests/tests/gr-pitr-minio/02-assert.yaml similarity index 87% rename from e2e-tests/tests/pitr-minio/02-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/02-assert.yaml index 10149fb52..b17d34c55 100644 --- a/e2e-tests/tests/pitr-minio/02-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/02-assert.yaml @@ -5,7 +5,7 @@ timeout: 600 kind: StatefulSet apiVersion: apps/v1 metadata: - name: pitr-minio-mysql + name: gr-pitr-minio-mysql status: observedGeneration: 1 replicas: 3 @@ -17,7 +17,7 @@ status: kind: Deployment apiVersion: apps/v1 metadata: - name: pitr-minio-router + name: gr-pitr-minio-router status: observedGeneration: 1 replicas: 3 @@ -27,7 +27,7 @@ status: kind: StatefulSet apiVersion: apps/v1 metadata: - name: pitr-minio-binlog-server + name: gr-pitr-minio-binlog-server status: observedGeneration: 1 replicas: 1 @@ -39,7 +39,7 @@ status: apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL metadata: - name: pitr-minio + name: gr-pitr-minio finalizers: - percona.com/delete-mysql-pods-in-order status: diff --git a/e2e-tests/tests/pitr-minio/02-create-cluster.yaml b/e2e-tests/tests/gr-pitr-minio/02-create-cluster.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/02-create-cluster.yaml rename to e2e-tests/tests/gr-pitr-minio/02-create-cluster.yaml diff --git a/e2e-tests/tests/pitr-minio/03-write-data.yaml b/e2e-tests/tests/gr-pitr-minio/03-write-data.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/03-write-data.yaml rename to e2e-tests/tests/gr-pitr-minio/03-write-data.yaml diff --git a/e2e-tests/tests/pitr-minio/04-assert.yaml b/e2e-tests/tests/gr-pitr-minio/04-assert.yaml similarity index 87% rename from e2e-tests/tests/pitr-minio/04-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/04-assert.yaml index 2f575b227..e23e05955 100644 --- a/e2e-tests/tests/pitr-minio/04-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/04-assert.yaml @@ -5,7 +5,7 @@ timeout: 300 kind: PerconaServerMySQLBackup apiVersion: ps.percona.com/v1 metadata: - name: pitr-minio-backup + name: gr-pitr-minio-backup finalizers: - percona.com/delete-backup status: diff --git a/e2e-tests/tests/pitr-minio/04-create-backup.yaml b/e2e-tests/tests/gr-pitr-minio/04-create-backup.yaml similarity index 71% rename from e2e-tests/tests/pitr-minio/04-create-backup.yaml rename to e2e-tests/tests/gr-pitr-minio/04-create-backup.yaml index b1015177c..80aa9b5d1 100644 --- a/e2e-tests/tests/pitr-minio/04-create-backup.yaml +++ b/e2e-tests/tests/gr-pitr-minio/04-create-backup.yaml @@ -1,9 +1,9 @@ apiVersion: ps.percona.com/v1 kind: PerconaServerMySQLBackup metadata: - name: pitr-minio-backup + name: gr-pitr-minio-backup finalizers: - percona.com/delete-backup spec: - clusterName: pitr-minio + clusterName: gr-pitr-minio storageName: minio diff --git a/e2e-tests/tests/pitr-minio/05-write-more-data.yaml b/e2e-tests/tests/gr-pitr-minio/05-write-more-data.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/05-write-more-data.yaml rename to e2e-tests/tests/gr-pitr-minio/05-write-more-data.yaml diff --git a/e2e-tests/tests/pitr-minio/06-assert.yaml b/e2e-tests/tests/gr-pitr-minio/06-assert.yaml similarity index 88% rename from e2e-tests/tests/pitr-minio/06-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/06-assert.yaml index 22f03ce31..b50e013c5 100644 --- a/e2e-tests/tests/pitr-minio/06-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/06-assert.yaml @@ -5,7 +5,7 @@ timeout: 600 apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL metadata: - name: pitr-minio + name: gr-pitr-minio finalizers: - percona.com/delete-mysql-pods-in-order status: @@ -22,6 +22,6 @@ status: kind: PerconaServerMySQLRestore apiVersion: ps.percona.com/v1 metadata: - name: pitr-minio-restore + name: gr-pitr-minio-restore status: state: Succeeded diff --git a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/gr-pitr-minio/06-create-pitr-restore.yaml similarity index 90% rename from e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml rename to e2e-tests/tests/gr-pitr-minio/06-create-pitr-restore.yaml index eba94939f..034372f2d 100644 --- a/e2e-tests/tests/pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/gr-pitr-minio/06-create-pitr-restore.yaml @@ -31,9 +31,9 @@ commands: echo '{}' \ | yq eval '.apiVersion = "ps.percona.com/v1"' - \ | yq eval '.kind = "PerconaServerMySQLRestore"' - \ - | yq eval '.metadata.name = "pitr-minio-restore"' - \ + | yq eval '.metadata.name = "gr-pitr-minio-restore"' - \ | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ - | yq eval '.spec.backupName = "pitr-minio-backup"' - \ + | yq eval '.spec.backupName = "gr-pitr-minio-backup"' - \ | yq eval '.spec.pitr.type = "gtid"' - \ | yq eval ".spec.pitr.gtid = \"${PITR_GTID}\"" - \ | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/pitr-minio/07-assert.yaml b/e2e-tests/tests/gr-pitr-minio/07-assert.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/07-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/07-assert.yaml diff --git a/e2e-tests/tests/pitr-minio/07-read-data.yaml b/e2e-tests/tests/gr-pitr-minio/07-read-data.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/07-read-data.yaml rename to e2e-tests/tests/gr-pitr-minio/07-read-data.yaml diff --git a/e2e-tests/tests/pitr-minio/08-assert.yaml b/e2e-tests/tests/gr-pitr-minio/08-assert.yaml similarity index 87% rename from e2e-tests/tests/pitr-minio/08-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/08-assert.yaml index bacd2174b..f83683ead 100644 --- a/e2e-tests/tests/pitr-minio/08-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/08-assert.yaml @@ -5,7 +5,7 @@ timeout: 600 apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL metadata: - name: pitr-minio + name: gr-pitr-minio finalizers: - percona.com/delete-mysql-pods-in-order status: @@ -22,6 +22,6 @@ status: kind: PerconaServerMySQLRestore apiVersion: ps.percona.com/v1 metadata: - name: pitr-minio-restore-date + name: gr-pitr-minio-restore-date status: state: Succeeded diff --git a/e2e-tests/tests/pitr-minio/08-create-date-restore.yaml b/e2e-tests/tests/gr-pitr-minio/08-create-date-restore.yaml similarity index 82% rename from e2e-tests/tests/pitr-minio/08-create-date-restore.yaml rename to e2e-tests/tests/gr-pitr-minio/08-create-date-restore.yaml index 601ca5f53..4d59a212f 100644 --- a/e2e-tests/tests/pitr-minio/08-create-date-restore.yaml +++ b/e2e-tests/tests/gr-pitr-minio/08-create-date-restore.yaml @@ -15,9 +15,9 @@ commands: echo '{}' \ | yq eval '.apiVersion = "ps.percona.com/v1"' - \ | yq eval '.kind = "PerconaServerMySQLRestore"' - \ - | yq eval '.metadata.name = "pitr-minio-restore-date"' - \ + | yq eval '.metadata.name = "gr-pitr-minio-restore-date"' - \ | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ - | yq eval '.spec.backupName = "pitr-minio-backup"' - \ + | yq eval '.spec.backupName = "gr-pitr-minio-backup"' - \ | yq eval '.spec.pitr.type = "date"' - \ | yq eval ".spec.pitr.date = \"${PITR_DATE}\"" - \ | kubectl apply -n "${NAMESPACE}" -f - \ No newline at end of file diff --git a/e2e-tests/tests/pitr-minio/09-assert.yaml b/e2e-tests/tests/gr-pitr-minio/09-assert.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/09-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/09-assert.yaml diff --git a/e2e-tests/tests/pitr-minio/09-read-data.yaml b/e2e-tests/tests/gr-pitr-minio/09-read-data.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/09-read-data.yaml rename to e2e-tests/tests/gr-pitr-minio/09-read-data.yaml diff --git a/e2e-tests/tests/pitr-minio/10-failover.yaml b/e2e-tests/tests/gr-pitr-minio/10-failover.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/10-failover.yaml rename to e2e-tests/tests/gr-pitr-minio/10-failover.yaml diff --git a/e2e-tests/tests/pitr-minio/11-assert.yaml b/e2e-tests/tests/gr-pitr-minio/11-assert.yaml similarity index 88% rename from e2e-tests/tests/pitr-minio/11-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/11-assert.yaml index df777fbcb..27c3468e6 100644 --- a/e2e-tests/tests/pitr-minio/11-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/11-assert.yaml @@ -5,7 +5,7 @@ timeout: 600 apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL metadata: - name: pitr-minio + name: gr-pitr-minio finalizers: - percona.com/delete-mysql-pods-in-order status: @@ -22,6 +22,6 @@ status: kind: PerconaServerMySQLBackup apiVersion: ps.percona.com/v1 metadata: - name: pitr-minio-backup-2 + name: gr-pitr-minio-backup-2 status: state: Succeeded \ No newline at end of file diff --git a/e2e-tests/tests/pitr-minio/11-write-and-backup.yaml b/e2e-tests/tests/gr-pitr-minio/11-write-and-backup.yaml similarity index 91% rename from e2e-tests/tests/pitr-minio/11-write-and-backup.yaml rename to e2e-tests/tests/gr-pitr-minio/11-write-and-backup.yaml index 48ce03895..a25dd6dce 100644 --- a/e2e-tests/tests/pitr-minio/11-write-and-backup.yaml +++ b/e2e-tests/tests/gr-pitr-minio/11-write-and-backup.yaml @@ -17,7 +17,7 @@ commands: echo '{}' \ | yq eval '.apiVersion = "ps.percona.com/v1"' - \ | yq eval '.kind = "PerconaServerMySQLBackup"' - \ - | yq eval '.metadata.name = "pitr-minio-backup-2"' - \ + | yq eval '.metadata.name = "gr-pitr-minio-backup-2"' - \ | yq eval '.metadata.finalizers[0] = "percona.com/delete-backup"' - \ | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ | yq eval '.spec.storageName = "minio"' - \ diff --git a/e2e-tests/tests/pitr-minio/12-assert.yaml b/e2e-tests/tests/gr-pitr-minio/12-assert.yaml similarity index 86% rename from e2e-tests/tests/pitr-minio/12-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/12-assert.yaml index b58bd3e0a..bebfc90f6 100644 --- a/e2e-tests/tests/pitr-minio/12-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/12-assert.yaml @@ -5,7 +5,7 @@ timeout: 600 apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL metadata: - name: pitr-minio + name: gr-pitr-minio finalizers: - percona.com/delete-mysql-pods-in-order status: @@ -22,6 +22,6 @@ status: kind: PerconaServerMySQLRestore apiVersion: ps.percona.com/v1 metadata: - name: pitr-minio-restore-post-failover + name: gr-pitr-minio-restore-post-failover status: state: Succeeded \ No newline at end of file diff --git a/e2e-tests/tests/pitr-minio/12-write-and-restore.yaml b/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml similarity index 85% rename from e2e-tests/tests/pitr-minio/12-write-and-restore.yaml rename to e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml index d69a9c375..556a565c3 100644 --- a/e2e-tests/tests/pitr-minio/12-write-and-restore.yaml +++ b/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml @@ -22,9 +22,9 @@ commands: echo '{}' \ | yq eval '.apiVersion = "ps.percona.com/v1"' - \ | yq eval '.kind = "PerconaServerMySQLRestore"' - \ - | yq eval '.metadata.name = "pitr-minio-restore-post-failover"' - \ + | yq eval '.metadata.name = "gr-pitr-minio-restore-post-failover"' - \ | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ - | yq eval '.spec.backupName = "pitr-minio-backup-2"' - \ + | yq eval '.spec.backupName = "gr-pitr-minio-backup-2"' - \ | yq eval '.spec.pitr.type = "date"' - \ | yq eval ".spec.pitr.date = \"${PITR_DATE_POST}\"" - \ | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/pitr-minio/13-assert.yaml b/e2e-tests/tests/gr-pitr-minio/13-assert.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/13-assert.yaml rename to e2e-tests/tests/gr-pitr-minio/13-assert.yaml diff --git a/e2e-tests/tests/pitr-minio/13-read-data.yaml b/e2e-tests/tests/gr-pitr-minio/13-read-data.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/13-read-data.yaml rename to e2e-tests/tests/gr-pitr-minio/13-read-data.yaml diff --git a/e2e-tests/tests/pitr-minio/14-switchover.yaml b/e2e-tests/tests/gr-pitr-minio/14-switchover.yaml similarity index 100% rename from e2e-tests/tests/pitr-minio/14-switchover.yaml rename to e2e-tests/tests/gr-pitr-minio/14-switchover.yaml diff --git a/e2e-tests/tests/pitr-minio/98-drop-finalizer.yaml b/e2e-tests/tests/gr-pitr-minio/98-drop-finalizer.yaml similarity index 78% rename from e2e-tests/tests/pitr-minio/98-drop-finalizer.yaml rename to e2e-tests/tests/gr-pitr-minio/98-drop-finalizer.yaml index b235ac0b7..70c1c6db6 100644 --- a/e2e-tests/tests/pitr-minio/98-drop-finalizer.yaml +++ b/e2e-tests/tests/gr-pitr-minio/98-drop-finalizer.yaml @@ -1,5 +1,5 @@ apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL metadata: - name: pitr-minio + name: gr-pitr-minio finalizers: [] diff --git a/e2e-tests/tests/pitr-minio/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/gr-pitr-minio/99-remove-cluster-gracefully.yaml similarity index 91% rename from e2e-tests/tests/pitr-minio/99-remove-cluster-gracefully.yaml rename to e2e-tests/tests/gr-pitr-minio/99-remove-cluster-gracefully.yaml index 3ebb90e80..35b44e567 100644 --- a/e2e-tests/tests/pitr-minio/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/gr-pitr-minio/99-remove-cluster-gracefully.yaml @@ -4,7 +4,7 @@ delete: - apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL metadata: - name: pitr-minio + name: gr-pitr-minio commands: - script: |- set -o errexit From ccebfc466d6173c5d8ed9b627d9a7f9735824f52 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 12:41:58 +0300 Subject: [PATCH 053/102] add configurable options for binlog server + unit test and default values --- api/v1/perconaservermysql_types.go | 30 +++++++++ api/v1/perconaservermysql_types_test.go | 63 +++++++++++++++++++ api/v1/zz_generated.deepcopy.go | 5 ++ .../ps.percona.com_perconaservermysqls.yaml | 10 +++ deploy/bundle.yaml | 10 +++ deploy/cr.yaml | 5 ++ deploy/crd.yaml | 10 +++ deploy/cw-bundle.yaml | 10 +++ pkg/controller/ps/controller.go | 10 +-- 9 files changed, 148 insertions(+), 5 deletions(-) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index 5cc2b91be..f821fd32b 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -533,6 +533,16 @@ type BinlogServerSpec struct { ServerID int32 `json:"serverId,omitempty"` // The number of seconds the utility will spend in disconnected mode between reconnection attempts. IdleTime int32 `json:"idleTime,omitempty"` + // SSLMode specifies the SSL mode for the connection to MySQL. Defaults to "verify_identity". + SSLMode string `json:"sslMode,omitempty"` + // VerifyChecksum enables checksum verification during replication. Defaults to true. + VerifyChecksum *bool `json:"verifyChecksum,omitempty"` + // RewriteFileSize specifies the maximum binlog file size for rewrite. Defaults to "128M". + RewriteFileSize string `json:"rewriteFileSize,omitempty"` + // CheckpointSize specifies the storage checkpoint size. Defaults to "2M". + CheckpointSize string `json:"checkpointSize,omitempty"` + // CheckpointInterval specifies the storage checkpoint interval. Defaults to "30s". + CheckpointInterval string `json:"checkpointInterval,omitempty"` PodSpec `json:",inline"` } @@ -995,6 +1005,26 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion cr.Spec.Backup.PiTR.BinlogServer = new(BinlogServerSpec) } + if cr.Spec.Backup.PiTR.BinlogServer != nil { + bls := cr.Spec.Backup.PiTR.BinlogServer + if bls.SSLMode == "" { + bls.SSLMode = "verify_identity" + } + if bls.VerifyChecksum == nil { + t := true + bls.VerifyChecksum = &t + } + if bls.RewriteFileSize == "" { + bls.RewriteFileSize = "128M" + } + if bls.CheckpointSize == "" { + bls.CheckpointSize = "2M" + } + if bls.CheckpointInterval == "" { + bls.CheckpointInterval = "30s" + } + } + if cr.Spec.Pause { cr.Spec.MySQL.Size = 0 cr.Spec.Orchestrator.Size = 0 diff --git a/api/v1/perconaservermysql_types_test.go b/api/v1/perconaservermysql_types_test.go index 2792b5f20..920dce18d 100644 --- a/api/v1/perconaservermysql_types_test.go +++ b/api/v1/perconaservermysql_types_test.go @@ -166,6 +166,69 @@ func TestCheckNSetDefaults(t *testing.T) { err := cr.CheckNSetDefaults(t.Context(), nil) assert.NoError(t, err) }) + t.Run("binlog server defaults are set when binlogServer is configured", func(t *testing.T) { + cr := new(PerconaServerMySQL) + cr.Spec.MySQL.VolumeSpec = &VolumeSpec{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1G"), + }, + }, + }, + } + cr.Spec.Backup = &BackupSpec{ + PiTR: PiTRSpec{ + BinlogServer: &BinlogServerSpec{}, + }, + } + + err := cr.CheckNSetDefaults(t.Context(), nil) + assert.NoError(t, err) + + bls := cr.Spec.Backup.PiTR.BinlogServer + assert.Equal(t, "verify_identity", bls.SSLMode) + assert.NotNil(t, bls.VerifyChecksum) + assert.True(t, *bls.VerifyChecksum) + assert.Equal(t, "128M", bls.RewriteFileSize) + assert.Equal(t, "2M", bls.CheckpointSize) + assert.Equal(t, "30s", bls.CheckpointInterval) + }) + t.Run("binlog server explicit values are not overridden by defaults", func(t *testing.T) { + cr := new(PerconaServerMySQL) + cr.Spec.MySQL.VolumeSpec = &VolumeSpec{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1G"), + }, + }, + }, + } + f := false + cr.Spec.Backup = &BackupSpec{ + PiTR: PiTRSpec{ + BinlogServer: &BinlogServerSpec{ + SSLMode: "required", + VerifyChecksum: &f, + RewriteFileSize: "256M", + CheckpointSize: "4M", + CheckpointInterval: "60s", + }, + }, + } + + err := cr.CheckNSetDefaults(t.Context(), nil) + assert.NoError(t, err) + + bls := cr.Spec.Backup.PiTR.BinlogServer + assert.Equal(t, "required", bls.SSLMode) + assert.NotNil(t, bls.VerifyChecksum) + assert.False(t, *bls.VerifyChecksum) + assert.Equal(t, "256M", bls.RewriteFileSize) + assert.Equal(t, "4M", bls.CheckpointSize) + assert.Equal(t, "60s", bls.CheckpointInterval) + }) } func TestValidateVolume(t *testing.T) { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 2e90a3cf9..85e4e4dc2 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -304,6 +304,11 @@ func (in *BackupStorageSpec) DeepCopy() *BackupStorageSpec { func (in *BinlogServerSpec) DeepCopyInto(out *BinlogServerSpec) { *out = *in in.Storage.DeepCopyInto(&out.Storage) + if in.VerifyChecksum != nil { + in, out := &in.VerifyChecksum, &out.VerifyChecksum + *out = new(bool) + **out = **in + } in.PodSpec.DeepCopyInto(&out.PodSpec) } diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 83dad6409..faecca96d 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -709,6 +709,10 @@ spec: additionalProperties: type: string type: object + checkpointInterval: + type: string + checkpointSize: + type: string configuration: type: string connectTimeout: @@ -1325,6 +1329,8 @@ spec: x-kubernetes-int-or-string: true type: object type: object + rewriteFileSize: + type: string runtimeClassName: type: string schedulerName: @@ -1337,6 +1343,8 @@ spec: size: format: int32 type: integer + sslMode: + type: string startupProbe: properties: exec: @@ -1507,6 +1515,8 @@ spec: - whenUnsatisfiable type: object type: array + verifyChecksum: + type: boolean writeTimeout: format: int32 type: integer diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 414f6457b..bde1af7b8 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -3052,6 +3052,10 @@ spec: additionalProperties: type: string type: object + checkpointInterval: + type: string + checkpointSize: + type: string configuration: type: string connectTimeout: @@ -3668,6 +3672,8 @@ spec: x-kubernetes-int-or-string: true type: object type: object + rewriteFileSize: + type: string runtimeClassName: type: string schedulerName: @@ -3680,6 +3686,8 @@ spec: size: format: int32 type: integer + sslMode: + type: string startupProbe: properties: exec: @@ -3850,6 +3858,8 @@ spec: - whenUnsatisfiable type: object type: array + verifyChecksum: + type: boolean writeTimeout: format: int32 type: integer diff --git a/deploy/cr.yaml b/deploy/cr.yaml index debf93c92..8b445d341 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -698,6 +698,8 @@ spec: # - e2e-az1 # - e2e-az2 # antiAffinityTopologyKey: kubernetes.io/hostname +# checkpointInterval: "" +# checkpointSize: "" # configuration: "" # containerSecurityContext: # privileged: false @@ -745,6 +747,8 @@ spec: # requests: # cpu: 200m # memory: 200M +# rewriteFileSize: "" +# sslMode: "" # startupProbe: # failureThreshold: 3 # periodSeconds: 5 @@ -762,6 +766,7 @@ spec: # maxSkew: 1 # topologyKey: kubernetes.io/hostname # whenUnsatisfiable: DoNotSchedule +# verifyChecksum: null # sourcePod: ps-cluster1-mysql-1 image: perconalab/percona-server-mysql-operator:main-backup8.4 imagePullPolicy: Always diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 4ea465697..f4a60e3d4 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -3052,6 +3052,10 @@ spec: additionalProperties: type: string type: object + checkpointInterval: + type: string + checkpointSize: + type: string configuration: type: string connectTimeout: @@ -3668,6 +3672,8 @@ spec: x-kubernetes-int-or-string: true type: object type: object + rewriteFileSize: + type: string runtimeClassName: type: string schedulerName: @@ -3680,6 +3686,8 @@ spec: size: format: int32 type: integer + sslMode: + type: string startupProbe: properties: exec: @@ -3850,6 +3858,8 @@ spec: - whenUnsatisfiable type: object type: array + verifyChecksum: + type: boolean writeTimeout: format: int32 type: integer diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 764918915..8e1fa1325 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -3052,6 +3052,10 @@ spec: additionalProperties: type: string type: object + checkpointInterval: + type: string + checkpointSize: + type: string configuration: type: string connectTimeout: @@ -3668,6 +3672,8 @@ spec: x-kubernetes-int-or-string: true type: object type: object + rewriteFileSize: + type: string runtimeClassName: type: string schedulerName: @@ -3680,6 +3686,8 @@ spec: size: format: int32 type: integer + sslMode: + type: string startupProbe: properties: exec: @@ -3850,6 +3858,8 @@ spec: - whenUnsatisfiable type: object type: array + verifyChecksum: + type: boolean writeTimeout: format: int32 type: integer diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index 52d92f23b..997a8eea4 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1345,7 +1345,7 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context WriteTimeout: cr.Spec.Backup.PiTR.BinlogServer.WriteTimeout, ReadTimeout: cr.Spec.Backup.PiTR.BinlogServer.ReadTimeout, SSL: &binlogserver.ConnectionSSL{ - Mode: "verify_identity", + Mode: cr.Spec.Backup.PiTR.BinlogServer.SSLMode, CA: path.Join(binlogserver.TLSMountPath, "ca.crt"), Cert: path.Join(binlogserver.TLSMountPath, "tls.crt"), Key: path.Join(binlogserver.TLSMountPath, "tls.key"), @@ -1355,17 +1355,17 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context Mode: binlogserver.ReplicationModeGTID, ServerID: cr.Spec.Backup.PiTR.BinlogServer.ServerID, IdleTime: cr.Spec.Backup.PiTR.BinlogServer.IdleTime, - VerifyChecksum: true, + VerifyChecksum: *cr.Spec.Backup.PiTR.BinlogServer.VerifyChecksum, Rewrite: binlogserver.Rewrite{ BaseFileName: "binlog", - FileSize: "128M", + FileSize: cr.Spec.Backup.PiTR.BinlogServer.RewriteFileSize, }, }, Storage: binlogserver.Storage{ Backend: "s3", URI: s3Uri, - CheckpointSize: "2M", - CheckpointInterval: "30s", + CheckpointSize: cr.Spec.Backup.PiTR.BinlogServer.CheckpointSize, + CheckpointInterval: cr.Spec.Backup.PiTR.BinlogServer.CheckpointInterval, FsBufferDirectory: binlogserver.BufferMountPath, }, } From 11f6e1678ddac440187c2d2ad1a0c1329f480f6a Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 12:51:32 +0300 Subject: [PATCH 054/102] fix linter --- pkg/binlogserver/binlog_server_test.go | 4 ++-- pkg/binlogserver/search_test.go | 2 +- pkg/pitr/pitr_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/binlogserver/binlog_server_test.go b/pkg/binlogserver/binlog_server_test.go index 37a832fd3..894b851f4 100644 --- a/pkg/binlogserver/binlog_server_test.go +++ b/pkg/binlogserver/binlog_server_test.go @@ -50,8 +50,8 @@ func TestStatefulSet(t *testing.T) { verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { sts := StatefulSet(cr, "init:latest", "abc123") - assert.Equal(t, "apps/v1", sts.TypeMeta.APIVersion) - assert.Equal(t, "StatefulSet", sts.TypeMeta.Kind) + assert.Equal(t, "apps/v1", sts.APIVersion) + assert.Equal(t, "StatefulSet", sts.Kind) assert.Equal(t, "my-cluster-binlog-server", sts.Name) assert.Equal(t, "test-ns", sts.Namespace) }, diff --git a/pkg/binlogserver/search_test.go b/pkg/binlogserver/search_test.go index 2311e0f99..6e6262c6e 100644 --- a/pkg/binlogserver/search_test.go +++ b/pkg/binlogserver/search_test.go @@ -35,7 +35,7 @@ func (f *fakeExecClient) Exec(_ context.Context, _ *corev1.Pod, _ string, cmd [] } if stdout != nil && f.response != nil { data, _ := json.Marshal(f.response) - stdout.Write(data) + _, _ = stdout.Write(data) } return nil } diff --git a/pkg/pitr/pitr_test.go b/pkg/pitr/pitr_test.go index a402e8eab..6cf787d0f 100644 --- a/pkg/pitr/pitr_test.go +++ b/pkg/pitr/pitr_test.go @@ -48,8 +48,8 @@ func TestRestoreJob(t *testing.T) { verify: func(t *testing.T, job *batchv1.Job) { assert.Equal(t, "pitr-restore-my-restore", job.Name) assert.Equal(t, "test-ns", job.Namespace) - assert.Equal(t, "batch/v1", job.TypeMeta.APIVersion) - assert.Equal(t, "Job", job.TypeMeta.Kind) + assert.Equal(t, "batch/v1", job.APIVersion) + assert.Equal(t, "Job", job.Kind) }, }, "job spec parallelism and completions": { From 204fc59746337353b0240d03e1f748c3a66710a4 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 13:01:08 +0300 Subject: [PATCH 055/102] small background fix --- pkg/controller/ps/status_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/controller/ps/status_test.go b/pkg/controller/ps/status_test.go index f73342fd6..ab83de3a5 100644 --- a/pkg/controller/ps/status_test.go +++ b/pkg/controller/ps/status_test.go @@ -1268,8 +1268,6 @@ func makeFakeReadyPods(cr *apiv1.PerconaServerMySQL, amount int, podType string) } func TestReconcileStatusBinlogServer(t *testing.T) { - ctx := context.Background() - cr, err := readDefaultCR("ps-cluster1", "status-1") require.NoError(t, err) cr.Spec.MySQL.ClusterType = apiv1.ClusterTypeAsync @@ -1437,8 +1435,8 @@ func TestReconcileStatusBinlogServer(t *testing.T) { }, } - require.NoError(t, r.reconcileCRStatus(ctx, cr, nil)) - require.NoError(t, r.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.Name}, cr)) + require.NoError(t, r.reconcileCRStatus(t.Context(), cr, nil)) + require.NoError(t, r.Get(t.Context(), types.NamespacedName{Namespace: cr.Namespace, Name: cr.Name}, cr)) opt := cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "Message") assert.Empty(t, cmp.Diff(cr.Status, tt.expected, opt)) From 2b3d10dba224e6a1a92b74e35344c5f55fa86a70 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 2 Apr 2026 19:47:55 +0300 Subject: [PATCH 056/102] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- pkg/pitr/pitr_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/pitr/pitr_test.go b/pkg/pitr/pitr_test.go index 6cf787d0f..e0e6dd6d3 100644 --- a/pkg/pitr/pitr_test.go +++ b/pkg/pitr/pitr_test.go @@ -503,10 +503,10 @@ func TestJobName(t *testing.T) { restoreName string expected string }{ - "simple name": {restoreName: "my-restore", expected: "pitr-restore-my-restore"}, - "name with numbers": {restoreName: "restore-123", expected: "pitr-restore-restore-123"}, - "single word name": {restoreName: "restore", expected: "pitr-restore-restore"}, - "name with dots": {restoreName: "restore.v1", expected: "pitr-restore-restore.v1"}, + "simple name": {restoreName: "my-restore", expected: "pitr-restore-my-restore"}, + "name with numbers": {restoreName: "restore-123", expected: "pitr-restore-restore-123"}, + "single word name": {restoreName: "restore", expected: "pitr-restore-restore"}, + "name with dots": {restoreName: "restore.v1", expected: "pitr-restore-restore.v1"}, } for name, tt := range tests { @@ -521,9 +521,9 @@ func TestJobName(t *testing.T) { func TestBinlogsConfigMap(t *testing.T) { tests := map[string]struct { - cluster *apiv1.PerconaServerMySQL - restore *apiv1.PerconaServerMySQLRestore - verify func(t *testing.T, cm *corev1.ConfigMap) + cluster *apiv1.PerconaServerMySQL + restore *apiv1.PerconaServerMySQLRestore + verify func(t *testing.T, cm *corev1.ConfigMap) }{ "basic metadata": { cluster: &apiv1.PerconaServerMySQL{ From 6a3973bb93cd6f4b5b47d9935290bbd147412857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Fri, 3 Apr 2026 09:07:35 +0300 Subject: [PATCH 057/102] print GTID_EXECUTED before and after recovery --- build/run-pitr-restore.sh | 2 +- cmd/internal/db/db.go | 6 ++++++ cmd/pitr/main.go | 12 ++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh index f95f239ce..dfae2c189 100644 --- a/build/run-pitr-restore.sh +++ b/build/run-pitr-restore.sh @@ -22,4 +22,4 @@ done /opt/percona/pitr apply echo "Stopping mysqld" -mysqladmin -u operator -p$(/dev/null diff --git a/cmd/internal/db/db.go b/cmd/internal/db/db.go index dfc052892..94712b9d3 100644 --- a/cmd/internal/db/db.go +++ b/cmd/internal/db/db.go @@ -420,6 +420,12 @@ func (d *DB) StartReplicaUntilPosition(ctx context.Context, relayLogFile string, return errors.Wrap(err, "start replica until position") } +func (d *DB) GetGTIDExecuted(ctx context.Context) (string, error) { + var gtid string + err := d.db.QueryRowContext(ctx, "SELECT @@GTID_EXECUTED").Scan(>id) + return gtid, errors.Wrap(err, "get GTID_EXECUTED") +} + func (d *DB) SetGTIDNextAutomatic(ctx context.Context) error { _, err := d.db.ExecContext(ctx, "SET GTID_NEXT='AUTOMATIC'") return errors.Wrap(err, "set GTID_NEXT to AUTOMATIC") diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 898a2e228..50c9bce06 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -180,6 +180,12 @@ func runApply(ctx context.Context) error { firstRelayLog := fmt.Sprintf("%s-relay-bin.000001", hostname) + currentGTID, err := database.GetGTIDExecuted(ctx) + if err != nil { + return fmt.Errorf("get current GTID_EXECUTED: %w", err) + } + log.Printf("GTID_EXECUTED: %s", currentGTID) + log.Println("running 'CHANGE REPLICATION SOURCE'") if err := database.ChangeReplicationSourceRelay(ctx, firstRelayLog, 4); err != nil { return fmt.Errorf("change replication source: %w", err) @@ -205,6 +211,12 @@ func runApply(ctx context.Context) error { return errors.Wrap(err, "reset replication") } + currentGTID, err = database.GetGTIDExecuted(ctx) + if err != nil { + return fmt.Errorf("get GTID_EXECUTED after restore: %w", err) + } + log.Printf("GTID_EXECUTED: %s", currentGTID) + log.Println("setting GTID_NEXT to AUTOMATIC") if err := database.SetGTIDNextAutomatic(ctx); err != nil { return fmt.Errorf("set GTID_NEXT to AUTOMATIC: %w", err) From 3e2bff9b85ea8028db599354ba8e7976b529f34a Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 3 Apr 2026 12:40:31 +0300 Subject: [PATCH 058/102] small refactor on main to be testable --- cmd/pitr/main.go | 41 ++- cmd/pitr/main_test.go | 418 +++++++++++++++++++++++++ pkg/binlogserver/binlog_server_test.go | 4 +- 3 files changed, 450 insertions(+), 13 deletions(-) create mode 100644 cmd/pitr/main_test.go diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 50c9bce06..534554e5c 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -9,6 +9,7 @@ import ( "net/url" "os" "os/exec" + "path/filepath" "strings" "time" @@ -20,6 +21,21 @@ import ( "github.com/pkg/errors" ) +// Database defines the MySQL operations needed for PITR apply. +type Database interface { + ChangeReplicationSourceRelay(ctx context.Context, relayLogFile string, relayLogPos int) error + StartReplicaUntilGTID(ctx context.Context, gtid string) error + WaitReplicaSQLThreadStop(ctx context.Context, pollInterval time.Duration) error + StopReplication(ctx context.Context) error + ResetReplication(ctx context.Context) error + SetGTIDNextAutomatic(ctx context.Context) error + GetGTIDExecuted(ctx context.Context) (string, error) + Close() error +} + +type newStorageFn func(ctx context.Context, endpoint, accessKey, secretKey, bucket, prefix, region string, verifyTLS bool) (storage.Storage, error) +type newDatabaseFn func(ctx context.Context, params db.DBParams) (Database, error) + func main() { if len(os.Args) < 2 { log.Fatal("usage: pitr ") @@ -29,11 +45,14 @@ func main() { switch os.Args[1] { case "setup": - if err := runSetup(ctx); err != nil { + if err := runSetup(ctx, storage.NewS3, "/var/lib/mysql"); err != nil { log.Fatalf("setup failed: %v", err) } case "apply": - if err := runApply(ctx); err != nil { + newDB := func(ctx context.Context, params db.DBParams) (Database, error) { + return db.NewDatabase(ctx, params) + } + if err := runApply(ctx, newDB, utils.GetSecret, getLatestGTIDByDatetime, "/var/lib/mysql"); err != nil { log.Fatalf("apply failed: %v", err) } default: @@ -41,7 +60,7 @@ func main() { } } -func runSetup(ctx context.Context) error { +func runSetup(ctx context.Context, newS3 newStorageFn, mysqlDir string) error { binlogsPath := os.Getenv("BINLOGS_PATH") if binlogsPath == "" { return fmt.Errorf("BINLOGS_PATH is not set") @@ -73,7 +92,7 @@ func runSetup(ctx context.Context) error { bucket := os.Getenv("S3_BUCKET") verifyTLS := os.Getenv("VERIFY_TLS") != "false" - s3Client, err := storage.NewS3(ctx, endpoint, accessKey, secretKey, bucket, "", region, verifyTLS) + s3Client, err := newS3(ctx, endpoint, accessKey, secretKey, bucket, "", region, verifyTLS) if err != nil { return fmt.Errorf("create S3 client: %w", err) } @@ -81,7 +100,7 @@ func runSetup(ctx context.Context) error { var relayLogFiles []string for i, entry := range entries { relayLogName := fmt.Sprintf("%s-relay-bin.%06d", hostname, i+1) - relayLogPath := fmt.Sprintf("/var/lib/mysql/%s", relayLogName) + relayLogPath := filepath.Join(mysqlDir, relayLogName) objectKey, err := objectKeyFromURI(entry.URI, bucket) if err != nil { @@ -117,7 +136,7 @@ func runSetup(ctx context.Context) error { relayLogFiles = append(relayLogFiles, "./"+relayLogName) } - indexPath := fmt.Sprintf("/var/lib/mysql/%s-relay-bin.index", hostname) + indexPath := filepath.Join(mysqlDir, fmt.Sprintf("%s-relay-bin.index", hostname)) indexContent := strings.Join(relayLogFiles, "\n") + "\n" if err := os.WriteFile(indexPath, []byte(indexContent), 0644); err != nil { return fmt.Errorf("write relay log index: %w", err) @@ -127,7 +146,7 @@ func runSetup(ctx context.Context) error { return nil } -func runApply(ctx context.Context) error { +func runApply(ctx context.Context, newDB newDatabaseFn, getSecret func(apiv1.SystemUser) (string, error), getGTIDByDatetime func(string, string) (string, error), mysqlDir string) error { pitrType := os.Getenv("PITR_TYPE") pitrDate := os.Getenv("PITR_DATE") pitrGTID := os.Getenv("PITR_GTID") @@ -137,12 +156,12 @@ func runApply(ctx context.Context) error { return fmt.Errorf("get hostname: %w", err) } - operatorPass, err := utils.GetSecret(apiv1.UserOperator) + operatorPass, err := getSecret(apiv1.UserOperator) if err != nil { return fmt.Errorf("get operator password: %w", err) } - database, err := db.NewDatabase(ctx, db.DBParams{ + database, err := newDB(ctx, db.DBParams{ User: apiv1.UserOperator, Pass: operatorPass, Host: "127.0.0.1", @@ -168,10 +187,10 @@ func runApply(ctx context.Context) error { } lastRelayLog := fmt.Sprintf("%s-relay-bin.%06d", hostname, len(entries)) - lastRelayLogPath := fmt.Sprintf("/var/lib/mysql/%s", lastRelayLog) + lastRelayLogPath := filepath.Join(mysqlDir, lastRelayLog) if pitrType == "date" { - pitrGTID, err = getLatestGTIDByDatetime(lastRelayLogPath, pitrDate) + pitrGTID, err = getGTIDByDatetime(lastRelayLogPath, pitrDate) if err != nil { return fmt.Errorf("get latest GTID for date %s: %w", pitrDate, err) } diff --git a/cmd/pitr/main_test.go b/cmd/pitr/main_test.go new file mode 100644 index 000000000..e0e1ce68f --- /dev/null +++ b/cmd/pitr/main_test.go @@ -0,0 +1,418 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" + "github.com/percona/percona-server-mysql-operator/cmd/internal/db" + "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" + "github.com/percona/percona-server-mysql-operator/pkg/xtrabackup/storage" +) + +type fakeStorage struct { + objects map[string]string // key -> content + getErr error +} + +func (f *fakeStorage) GetObject(_ context.Context, objectName string) (io.ReadCloser, error) { + if f.getErr != nil { + return nil, f.getErr + } + content, ok := f.objects[objectName] + if !ok { + return nil, storage.ErrObjectNotFound + } + return io.NopCloser(strings.NewReader(content)), nil +} + +func (f *fakeStorage) PutObject(_ context.Context, _ string, _ io.Reader, _ int64) error { return nil } +func (f *fakeStorage) ListObjects(_ context.Context, _ string) ([]string, error) { return nil, nil } +func (f *fakeStorage) DeleteObject(_ context.Context, _ string) error { return nil } +func (f *fakeStorage) SetPrefix(_ string) {} +func (f *fakeStorage) GetPrefix() string { return "" } + +// fakeDB records method calls and returns configured errors. +type fakeDB struct { + changeRelayErr error + startUntilErr error + waitErr error + stopErr error + resetErr error + setGTIDNextErr error + getGTIDExecutedErr error + getGTIDExecutedResult string + calls []string + startUntilGTID string +} + +func (f *fakeDB) ChangeReplicationSourceRelay(_ context.Context, _ string, _ int) error { + f.calls = append(f.calls, "ChangeReplicationSourceRelay") + return f.changeRelayErr +} + +func (f *fakeDB) StartReplicaUntilGTID(_ context.Context, gtid string) error { + f.calls = append(f.calls, "StartReplicaUntilGTID") + f.startUntilGTID = gtid + return f.startUntilErr +} + +func (f *fakeDB) WaitReplicaSQLThreadStop(_ context.Context, _ time.Duration) error { + f.calls = append(f.calls, "WaitReplicaSQLThreadStop") + return f.waitErr +} + +func (f *fakeDB) StopReplication(_ context.Context) error { + f.calls = append(f.calls, "StopReplication") + return f.stopErr +} + +func (f *fakeDB) ResetReplication(_ context.Context) error { + f.calls = append(f.calls, "ResetReplication") + return f.resetErr +} + +func (f *fakeDB) SetGTIDNextAutomatic(_ context.Context) error { + f.calls = append(f.calls, "SetGTIDNextAutomatic") + return f.setGTIDNextErr +} + +func (f *fakeDB) GetGTIDExecuted(_ context.Context) (string, error) { + f.calls = append(f.calls, "GetGTIDExecuted") + return f.getGTIDExecutedResult, f.getGTIDExecutedErr +} + +func (f *fakeDB) Close() error { return nil } + +func writeBinlogsFile(t *testing.T, entries []binlogserver.BinlogEntry) string { + t.Helper() + data, err := json.Marshal(entries) + require.NoError(t, err) + f, err := os.CreateTemp(t.TempDir(), "binlogs-*.json") + require.NoError(t, err) + _, err = f.Write(data) + require.NoError(t, err) + require.NoError(t, f.Close()) + return f.Name() +} + +func TestRunSetup(t *testing.T) { + bucket := "mybucket" + + tests := map[string]struct { + setupEnv func(t *testing.T, binlogsPath string) + entries []binlogserver.BinlogEntry + rawContent string + newS3 func(*fakeStorage) newStorageFn + expectedError string + checkResult func(t *testing.T, mysqlDir string) + }{ + "missing BINLOGS_PATH": { + setupEnv: func(t *testing.T, _ string) { + t.Setenv("BINLOGS_PATH", "") + }, + expectedError: "BINLOGS_PATH", + }, + "invalid JSON in binlogs file": { + setupEnv: func(t *testing.T, binlogsPath string) { + t.Setenv("BINLOGS_PATH", binlogsPath) + }, + rawContent: "not-json", + expectedError: "parse binlogs json", + }, + "empty binlog entries": { + setupEnv: func(t *testing.T, binlogsPath string) { + t.Setenv("BINLOGS_PATH", binlogsPath) + }, + entries: []binlogserver.BinlogEntry{}, + expectedError: "no binlog entries found", + }, + "S3 client creation error": { + setupEnv: func(t *testing.T, binlogsPath string) { + t.Setenv("BINLOGS_PATH", binlogsPath) + t.Setenv("S3_BUCKET", bucket) + }, + entries: []binlogserver.BinlogEntry{ + {URI: "s3://mybucket/binlogs/binlog.000001"}, + }, + newS3: func(_ *fakeStorage) newStorageFn { + return func(_ context.Context, _, _, _, _, _, _ string, _ bool) (storage.Storage, error) { + return nil, errors.New("s3 unavailable") + } + }, + expectedError: "create S3 client", + }, + "GetObject error": { + setupEnv: func(t *testing.T, binlogsPath string) { + t.Setenv("BINLOGS_PATH", binlogsPath) + t.Setenv("S3_BUCKET", bucket) + }, + entries: []binlogserver.BinlogEntry{ + {URI: "s3://mybucket/binlogs/binlog.000001"}, + }, + newS3: func(fake *fakeStorage) newStorageFn { + fake.getErr = errors.New("download failed") + return func(_ context.Context, _, _, _, _, _, _ string, _ bool) (storage.Storage, error) { + return fake, nil + } + }, + expectedError: "download binlog", + }, + "success": { + setupEnv: func(t *testing.T, binlogsPath string) { + t.Setenv("BINLOGS_PATH", binlogsPath) + t.Setenv("S3_BUCKET", bucket) + }, + entries: []binlogserver.BinlogEntry{ + {URI: "s3://mybucket/binlogs/binlog.000001"}, + {URI: "s3://mybucket/binlogs/binlog.000002"}, + }, + newS3: func(fake *fakeStorage) newStorageFn { + fake.objects = map[string]string{ + "binlogs/binlog.000001": "binlogdata1", + "binlogs/binlog.000002": "binlogdata2", + } + return func(_ context.Context, _, _, _, _, _, _ string, _ bool) (storage.Storage, error) { + return fake, nil + } + }, + checkResult: func(t *testing.T, mysqlDir string) { + hostname, err := os.Hostname() + require.NoError(t, err) + + indexPath := filepath.Join(mysqlDir, hostname+"-relay-bin.index") + indexData, err := os.ReadFile(indexPath) + require.NoError(t, err, "relay log index file must exist") + + indexContent := string(indexData) + assert.Contains(t, indexContent, hostname+"-relay-bin.000001") + assert.Contains(t, indexContent, hostname+"-relay-bin.000002") + + for i, wantContent := range []string{"binlogdata1", "binlogdata2"} { + relayLog := filepath.Join(mysqlDir, fmt.Sprintf("%s-relay-bin.%06d", hostname, i+1)) + data, err := os.ReadFile(relayLog) + require.NoErrorf(t, err, "relay log %d must exist", i+1) + assert.Equalf(t, wantContent, string(data), "relay log %d content mismatch", i+1) + } + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + var binlogsPath string + if tc.rawContent != "" { + f, err := os.CreateTemp(t.TempDir(), "binlogs-*.json") + require.NoError(t, err) + _, err = f.WriteString(tc.rawContent) + require.NoError(t, err) + require.NoError(t, f.Close()) + binlogsPath = f.Name() + } else if tc.entries != nil { + binlogsPath = writeBinlogsFile(t, tc.entries) + } + + tc.setupEnv(t, binlogsPath) + + fake := &fakeStorage{} + var newS3 newStorageFn + if tc.newS3 != nil { + newS3 = tc.newS3(fake) + } + + mysqlDir := t.TempDir() + err := runSetup(t.Context(), newS3, mysqlDir) + + if tc.expectedError != "" { + require.ErrorContains(t, err, tc.expectedError) + return + } + require.NoError(t, err) + if tc.checkResult != nil { + tc.checkResult(t, mysqlDir) + } + }) + } +} + +func TestRunApply(t *testing.T) { + defaultEntries := []binlogserver.BinlogEntry{ + {URI: "s3://bucket/binlogs/binlog.000001"}, + {URI: "s3://bucket/binlogs/binlog.000002"}, + } + allDBCalls := []string{ + "GetGTIDExecuted", + "ChangeReplicationSourceRelay", + "StartReplicaUntilGTID", + "WaitReplicaSQLThreadStop", + "StopReplication", + "ResetReplication", + "GetGTIDExecuted", + "SetGTIDNextAutomatic", + } + + tests := map[string]struct { + entries []binlogserver.BinlogEntry + pitrType string + pitrGTID string + pitrDate string + db *fakeDB + newDB func(ctx context.Context, params db.DBParams) (Database, error) + getSecret func(apiv1.SystemUser) (string, error) + getGTID func(string, string) (string, error) + expectedError string + expectedFuncCalls []string + expectedUDID string + }{ + "get secret error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + getSecret: func(apiv1.SystemUser) (string, error) { return "", errors.New("secret not found") }, + expectedError: "get operator password", + }, + "DB connect error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + newDB: func(_ context.Context, _ db.DBParams) (Database, error) { + return nil, errors.New("connection refused") + }, + expectedError: "connect to MySQL", + }, + "change replication source relay error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{changeRelayErr: errors.New("relay error")}, + expectedError: "change replication source", + expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay"}, + }, + "start replica until GTID error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{startUntilErr: errors.New("start error")}, + expectedError: "start replica until GTID", + expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay", "StartReplicaUntilGTID"}, + }, + "wait replica stop error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{waitErr: errors.New("wait error")}, + expectedError: "wait for replication", + expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay", "StartReplicaUntilGTID", "WaitReplicaSQLThreadStop"}, + }, + "stop replication error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{stopErr: errors.New("stop error")}, + expectedError: "stop replication", + expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay", "StartReplicaUntilGTID", "WaitReplicaSQLThreadStop", "StopReplication"}, + }, + "reset replication error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{resetErr: errors.New("reset error")}, + expectedError: "reset replication", + expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay", "StartReplicaUntilGTID", "WaitReplicaSQLThreadStop", "StopReplication", "ResetReplication"}, + }, + "set GTID_NEXT error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{setGTIDNextErr: errors.New("gtid error")}, + expectedError: "set GTID_NEXT to AUTOMATIC", + expectedFuncCalls: allDBCalls, + }, + "GTID mode success": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "aaaaaaaa-0000-0000-0000-000000000001:1-10", + db: &fakeDB{}, + expectedFuncCalls: allDBCalls, + expectedUDID: "aaaaaaaa-0000-0000-0000-000000000001:1-10", + }, + "date mode success": { + entries: defaultEntries, + pitrType: "date", + pitrDate: "2024-01-15 12:00:00", + db: &fakeDB{}, + getGTID: func(_ string, datetime string) (string, error) { + assert.Equal(t, "2024-01-15 12:00:00", datetime) + return "bbbbbbbb-0000-0000-0000-000000000002:1-5", nil + }, + expectedFuncCalls: allDBCalls, + expectedUDID: "bbbbbbbb-0000-0000-0000-000000000002:1-5", + }, + "date mode getGTID error": { + entries: defaultEntries, + pitrType: "date", + pitrDate: "2024-01-15 12:00:00", + db: &fakeDB{}, + getGTID: func(string, string) (string, error) { return "", errors.New("mysqlbinlog failed") }, + expectedError: "get latest GTID for date", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + binlogsPath := writeBinlogsFile(t, tc.entries) + t.Setenv("BINLOGS_PATH", binlogsPath) + t.Setenv("PITR_TYPE", tc.pitrType) + t.Setenv("PITR_GTID", tc.pitrGTID) + t.Setenv("PITR_DATE", tc.pitrDate) + + fakeDatabase := tc.db + + newDB := tc.newDB + if newDB == nil { + newDB = func(_ context.Context, _ db.DBParams) (Database, error) { + return fakeDatabase, nil + } + } + + getSecret := tc.getSecret + if getSecret == nil { + getSecret = func(apiv1.SystemUser) (string, error) { return "testpass", nil } + } + + getGTID := tc.getGTID + if getGTID == nil { + getGTID = func(string, string) (string, error) { + t.Fatal("getGTIDByDatetime should not be called") + return "", nil + } + } + + err := runApply(t.Context(), newDB, getSecret, getGTID, t.TempDir()) + + if tc.expectedError != "" { + require.ErrorContains(t, err, tc.expectedError) + } else { + require.NoError(t, err) + } + + if tc.expectedFuncCalls != nil && fakeDatabase != nil { + assert.Equal(t, tc.expectedFuncCalls, fakeDatabase.calls) + } + if tc.expectedUDID != "" && fakeDatabase != nil { + assert.Equal(t, tc.expectedUDID, fakeDatabase.startUntilGTID) + } + }) + } +} diff --git a/pkg/binlogserver/binlog_server_test.go b/pkg/binlogserver/binlog_server_test.go index 894b851f4..10ef17da9 100644 --- a/pkg/binlogserver/binlog_server_test.go +++ b/pkg/binlogserver/binlog_server_test.go @@ -79,10 +79,10 @@ func TestStatefulSet(t *testing.T) { "config hash annotation in pod template": { cr: newTestCR("cluster", "ns"), initImage: "init:latest", - configHash: "deadbeef", + configHash: "2pacisnotdead", verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { sts := StatefulSet(cr, "init:latest", "2pacisnotdead") - assert.Equal(t, "deadbeef", sts.Spec.Template.Annotations[string(naming.AnnotationConfigHash)]) + assert.Equal(t, "2pacisnotdead", sts.Spec.Template.Annotations[string(naming.AnnotationConfigHash)]) }, }, "empty config hash produces no annotation": { From d9b78065279af4160d51d58e6b8c3ae0e0e04510 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 3 Apr 2026 13:04:00 +0300 Subject: [PATCH 059/102] Update cmd/pitr/main_test.go Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- cmd/pitr/main_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/pitr/main_test.go b/cmd/pitr/main_test.go index e0e1ce68f..cefe88a52 100644 --- a/cmd/pitr/main_test.go +++ b/cmd/pitr/main_test.go @@ -53,8 +53,8 @@ type fakeDB struct { setGTIDNextErr error getGTIDExecutedErr error getGTIDExecutedResult string - calls []string - startUntilGTID string + calls []string + startUntilGTID string } func (f *fakeDB) ChangeReplicationSourceRelay(_ context.Context, _ string, _ int) error { From 7584be23dc12027a58938771ada9dd2d1f6f244a Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 3 Apr 2026 13:04:24 +0300 Subject: [PATCH 060/102] Update cmd/pitr/main_test.go Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- cmd/pitr/main_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/pitr/main_test.go b/cmd/pitr/main_test.go index cefe88a52..80020f2b7 100644 --- a/cmd/pitr/main_test.go +++ b/cmd/pitr/main_test.go @@ -45,13 +45,13 @@ func (f *fakeStorage) GetPrefix() string // fakeDB records method calls and returns configured errors. type fakeDB struct { - changeRelayErr error - startUntilErr error - waitErr error - stopErr error - resetErr error - setGTIDNextErr error - getGTIDExecutedErr error + changeRelayErr error + startUntilErr error + waitErr error + stopErr error + resetErr error + setGTIDNextErr error + getGTIDExecutedErr error getGTIDExecutedResult string calls []string startUntilGTID string From 8b3ace8027098e11dc69234391bc7073f06ac010 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 3 Apr 2026 17:22:52 +0300 Subject: [PATCH 061/102] add guard for bool pointer --- pkg/controller/ps/controller.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index 997a8eea4..6c71de9be 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1331,6 +1331,11 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context } configSecret.Data = make(map[string][]byte) + verifyChecksum := true + if cr.Spec.Backup.PiTR.BinlogServer.VerifyChecksum != nil { + verifyChecksum = *cr.Spec.Backup.PiTR.BinlogServer.VerifyChecksum + } + config := binlogserver.Configuration{ Logger: binlogserver.Logger{ Level: "debug", @@ -1355,7 +1360,7 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context Mode: binlogserver.ReplicationModeGTID, ServerID: cr.Spec.Backup.PiTR.BinlogServer.ServerID, IdleTime: cr.Spec.Backup.PiTR.BinlogServer.IdleTime, - VerifyChecksum: *cr.Spec.Backup.PiTR.BinlogServer.VerifyChecksum, + VerifyChecksum: verifyChecksum, Rewrite: binlogserver.Rewrite{ BaseFileName: "binlog", FileSize: cr.Spec.Backup.PiTR.BinlogServer.RewriteFileSize, From 42ca3501928e104e30afaca5937f2b6d3248ec95 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Fri, 3 Apr 2026 17:54:56 +0300 Subject: [PATCH 062/102] fix linting tests --- pkg/pitr/pitr_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pitr/pitr_test.go b/pkg/pitr/pitr_test.go index e0e6dd6d3..b3dd6d9b9 100644 --- a/pkg/pitr/pitr_test.go +++ b/pkg/pitr/pitr_test.go @@ -535,8 +535,8 @@ func TestBinlogsConfigMap(t *testing.T) { verify: func(t *testing.T, cm *corev1.ConfigMap) { assert.Equal(t, "pitr-binlogs-my-restore", cm.Name) assert.Equal(t, "test-ns", cm.Namespace) - assert.Equal(t, "v1", cm.TypeMeta.APIVersion) - assert.Equal(t, "ConfigMap", cm.TypeMeta.Kind) + assert.Equal(t, "v1", cm.APIVersion) + assert.Equal(t, "ConfigMap", cm.Kind) }, }, "no global labels or annotations": { From ffc3da6e89b76514bb9a73e4298dc1adc0efd56e Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Mon, 6 Apr 2026 18:46:55 +0300 Subject: [PATCH 063/102] handle disabled ssl mode --- api/v1/perconaservermysql_types.go | 1 + .../ps.percona.com_perconaservermysqls.yaml | 4 + deploy/bundle.yaml | 4 + deploy/crd.yaml | 4 + deploy/cw-bundle.yaml | 4 + pkg/binlogserver/binlog_server.go | 74 ++++++++++++------- pkg/binlogserver/binlog_server_test.go | 22 +++++- pkg/controller/ps/binlogserver_test.go | 59 +++++++++++++++ pkg/controller/ps/controller.go | 17 +++-- pkg/controller/ps/controller_test.go | 16 ++++ 10 files changed, 169 insertions(+), 36 deletions(-) create mode 100644 pkg/controller/ps/binlogserver_test.go diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index 293205653..616be8009 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -520,6 +520,7 @@ func (b *BackupStorageAzureSpec) ContainerAndPrefix() (string, string) { // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || has(self.binlogServer)",message="binlogServer is required when pitr is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image != '')",message="binlogServer.image is required when pitr is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size > 0)",message="binlogServer.size is required when pitr is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.serverId) && self.binlogServer.serverId > 0)",message="binlogServer.serverId is required when pitr is enabled" type PiTRSpec struct { Enabled bool `json:"enabled,omitempty"` diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 9ffa5366d..50112cbcd 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -1538,6 +1538,10 @@ spec: rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size > 0)' + - message: binlogServer.serverId is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.serverId) && self.binlogServer.serverId + > 0)' resources: properties: claims: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index f91f881c4..2d9654656 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -3901,6 +3901,10 @@ spec: rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size > 0)' + - message: binlogServer.serverId is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.serverId) && self.binlogServer.serverId + > 0)' resources: properties: claims: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index f57c2af04..6f974cddf 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -3901,6 +3901,10 @@ spec: rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size > 0)' + - message: binlogServer.serverId is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.serverId) && self.binlogServer.serverId + > 0)' resources: properties: claims: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index a07c34b01..f31b1899e 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -3901,6 +3901,10 @@ spec: rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size > 0)' + - message: binlogServer.serverId is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + || (has(self.binlogServer.serverId) && self.binlogServer.serverId + > 0)' resources: properties: claims: diff --git a/pkg/binlogserver/binlog_server.go b/pkg/binlogserver/binlog_server.go index df282dfcf..1d99aa8da 100644 --- a/pkg/binlogserver/binlog_server.go +++ b/pkg/binlogserver/binlog_server.go @@ -101,6 +101,10 @@ func StatefulSet(cr *apiv1.PerconaServerMySQL, initImage, configHash string) *ap } } +func sslDisabled(cr *apiv1.PerconaServerMySQL) bool { + return cr.Spec.Backup.PiTR.BinlogServer.SSLMode == "disabled" +} + func volumes(cr *apiv1.PerconaServerMySQL) []corev1.Volume { t := true @@ -108,7 +112,7 @@ func volumes(cr *apiv1.PerconaServerMySQL) []corev1.Volume { conf := Configurable(*cr) - return []corev1.Volume{ + vols := []corev1.Volume{ { Name: apiv1.BinVolumeName, VolumeSource: corev1.VolumeSource{ @@ -129,15 +133,21 @@ func volumes(cr *apiv1.PerconaServerMySQL) []corev1.Volume { }, }, }, - { + } + + if !sslDisabled(cr) { + vols = append(vols, corev1.Volume{ Name: tlsVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: cr.Spec.SSLSecretName, }, }, - }, - { + }) + } + + vols = append(vols, + corev1.Volume{ Name: storageCredsVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ @@ -145,7 +155,7 @@ func volumes(cr *apiv1.PerconaServerMySQL) []corev1.Volume { }, }, }, - { + corev1.Volume{ Name: configVolumeName, VolumeSource: corev1.VolumeSource{ Projected: &corev1.ProjectedVolumeSource{ @@ -181,7 +191,9 @@ func volumes(cr *apiv1.PerconaServerMySQL) []corev1.Volume { }, }, }, - } + ) + + return vols } func containers(cr *apiv1.PerconaServerMySQL) []corev1.Container { @@ -203,6 +215,33 @@ func binlogServerContainer(cr *apiv1.PerconaServerMySQL) corev1.Container { } env = append(env, spec.Env...) + mounts := []corev1.VolumeMount{ + { + Name: apiv1.BinVolumeName, + MountPath: apiv1.BinVolumePath, + }, + { + Name: credsVolumeName, + MountPath: CredsMountPath, + }, + } + if !sslDisabled(cr) { + mounts = append(mounts, corev1.VolumeMount{ + Name: tlsVolumeName, + MountPath: TLSMountPath, + }) + } + mounts = append(mounts, + corev1.VolumeMount{ + Name: configVolumeName, + MountPath: configMountPath, + }, + corev1.VolumeMount{ + Name: bufferVolumeName, + MountPath: BufferMountPath, + }, + ) + return corev1.Container{ Name: AppName, Image: spec.Image, @@ -210,28 +249,7 @@ func binlogServerContainer(cr *apiv1.PerconaServerMySQL) corev1.Container { Resources: spec.Resources, Env: env, EnvFrom: spec.EnvFrom, - VolumeMounts: []corev1.VolumeMount{ - { - Name: apiv1.BinVolumeName, - MountPath: apiv1.BinVolumePath, - }, - { - Name: credsVolumeName, - MountPath: CredsMountPath, - }, - { - Name: tlsVolumeName, - MountPath: TLSMountPath, - }, - { - Name: configVolumeName, - MountPath: configMountPath, - }, - { - Name: bufferVolumeName, - MountPath: BufferMountPath, - }, - }, + VolumeMounts: mounts, Command: []string{"/opt/percona/binlog-server-entrypoint.sh"}, Args: []string{binlogServerBinary, "pull", path.Join(configMountPath, ConfigKey)}, TerminationMessagePath: "/dev/termination-log", diff --git a/pkg/binlogserver/binlog_server_test.go b/pkg/binlogserver/binlog_server_test.go index 10ef17da9..3077e8c95 100644 --- a/pkg/binlogserver/binlog_server_test.go +++ b/pkg/binlogserver/binlog_server_test.go @@ -138,7 +138,7 @@ func TestStatefulSet(t *testing.T) { assert.Equal(t, "percona/binlog-server:2.0", container.Image) }, }, - "volumes include all expected volumes": { + "volumes include all expected volumes when ssl enabled": { cr: newTestCR("cluster", "ns"), initImage: "init:latest", verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { @@ -155,6 +155,24 @@ func TestStatefulSet(t *testing.T) { assert.True(t, volumeNames[configVolumeName], "missing config volume") }, }, + "tls volume and mount absent when ssl disabled": { + cr: func() *apiv1.PerconaServerMySQL { + cr := newTestCR("cluster", "ns") + cr.Spec.Backup.PiTR.BinlogServer.SSLMode = "disabled" + return cr + }(), + initImage: "init:latest", + verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { + sts := StatefulSet(cr, "init:latest", "") + for _, v := range sts.Spec.Template.Spec.Volumes { + assert.NotEqual(t, tlsVolumeName, v.Name, "tls volume should be absent when ssl is disabled") + } + container := sts.Spec.Template.Spec.Containers[0] + for _, m := range container.VolumeMounts { + assert.NotEqual(t, tlsVolumeName, m.Name, "tls volume mount should be absent when ssl is disabled") + } + }, + }, "creds volume uses internal secret name": { cr: newTestCR("mycluster", "ns"), initImage: "init:latest", @@ -250,7 +268,7 @@ func TestStatefulSet(t *testing.T) { assert.Equal(t, "prod", sts.Labels["env"]) }, }, - "container volume mounts include all expected mounts": { + "container volume mounts include all expected mounts when ssl enabled": { cr: newTestCR("cluster", "ns"), initImage: "init:latest", verify: func(t *testing.T, cr *apiv1.PerconaServerMySQL) { diff --git a/pkg/controller/ps/binlogserver_test.go b/pkg/controller/ps/binlogserver_test.go new file mode 100644 index 000000000..6257e3b8d --- /dev/null +++ b/pkg/controller/ps/binlogserver_test.go @@ -0,0 +1,59 @@ +package ps + +import ( + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" +) + +func TestBinlogServerSSLConfig(t *testing.T) { + tests := map[string]struct { + sslMode string + wantMode string + wantCerts bool + }{ + "disabled mode skips certificates": { + sslMode: "disabled", + wantMode: "disabled", + wantCerts: false, + }, + "verify_identity includes certificates": { + sslMode: "verify_identity", + wantMode: "verify_identity", + wantCerts: true, + }, + "verify_ca includes certificates": { + sslMode: "verify_ca", + wantMode: "verify_ca", + wantCerts: true, + }, + "required includes certificates": { + sslMode: "required", + wantMode: "required", + wantCerts: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := binlogServerSSLConfig(tt.sslMode) + require.NotNil(t, got) + + assert.Equal(t, tt.wantMode, got.Mode) + + if tt.wantCerts { + assert.Equal(t, path.Join(binlogserver.TLSMountPath, "ca.crt"), got.CA) + assert.Equal(t, path.Join(binlogserver.TLSMountPath, "tls.crt"), got.Cert) + assert.Equal(t, path.Join(binlogserver.TLSMountPath, "tls.key"), got.Key) + } else { + assert.Empty(t, got.CA) + assert.Empty(t, got.Cert) + assert.Empty(t, got.Key) + } + }) + } +} \ No newline at end of file diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index 6c71de9be..b8040a82d 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1284,6 +1284,16 @@ func (r *PerconaServerMySQLReconciler) reconcileMySQLRouter(ctx context.Context, return nil } +func binlogServerSSLConfig(sslMode string) *binlogserver.ConnectionSSL { + ssl := &binlogserver.ConnectionSSL{Mode: sslMode} + if sslMode != "disabled" { + ssl.CA = path.Join(binlogserver.TLSMountPath, "ca.crt") + ssl.Cert = path.Join(binlogserver.TLSMountPath, "tls.crt") + ssl.Key = path.Join(binlogserver.TLSMountPath, "tls.key") + } + return ssl +} + func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context, cr *apiv1.PerconaServerMySQL) error { if !cr.Spec.Backup.PiTR.Enabled { return nil @@ -1349,12 +1359,7 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context ConnectTimeout: cr.Spec.Backup.PiTR.BinlogServer.ConnectTimeout, WriteTimeout: cr.Spec.Backup.PiTR.BinlogServer.WriteTimeout, ReadTimeout: cr.Spec.Backup.PiTR.BinlogServer.ReadTimeout, - SSL: &binlogserver.ConnectionSSL{ - Mode: cr.Spec.Backup.PiTR.BinlogServer.SSLMode, - CA: path.Join(binlogserver.TLSMountPath, "ca.crt"), - Cert: path.Join(binlogserver.TLSMountPath, "tls.crt"), - Key: path.Join(binlogserver.TLSMountPath, "tls.key"), - }, + SSL: binlogServerSSLConfig(cr.Spec.Backup.PiTR.BinlogServer.SSLMode), }, Replication: binlogserver.Replication{ Mode: binlogserver.ReplicationModeGTID, diff --git a/pkg/controller/ps/controller_test.go b/pkg/controller/ps/controller_test.go index b63a1070c..b643fb9b6 100644 --- a/pkg/controller/ps/controller_test.go +++ b/pkg/controller/ps/controller_test.go @@ -1104,6 +1104,21 @@ var _ = Describe("CR validations", Ordered, func() { }) }) + When("pitr is enabled but binlogServer serverId is 0", Ordered, func() { + cr, err := readDefaultCR("pitr-enabled-no-serverid", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.Backup.PiTR.Enabled = true + cr.Spec.Backup.PiTR.BinlogServer = &psv1.BinlogServerSpec{} + cr.Spec.Backup.PiTR.BinlogServer.Image = "binlog-server-image" + cr.Spec.Backup.PiTR.BinlogServer.Size = 1 + It("should fail with serverId required error", func() { + createErr := k8sClient.Create(ctx, cr) + Expect(createErr).To(HaveOccurred()) + Expect(createErr.Error()).To(ContainSubstring("binlogServer.serverId is required when pitr is enabled")) + }) + }) + When("pitr is enabled with all required fields set", Ordered, func() { cr, err := readDefaultCR("pitr-enabled-valid", ns) Expect(err).NotTo(HaveOccurred()) @@ -1112,6 +1127,7 @@ var _ = Describe("CR validations", Ordered, func() { cr.Spec.Backup.PiTR.BinlogServer = &psv1.BinlogServerSpec{} cr.Spec.Backup.PiTR.BinlogServer.Image = "binlog-server-image" cr.Spec.Backup.PiTR.BinlogServer.Size = 1 + cr.Spec.Backup.PiTR.BinlogServer.ServerID = 100 It("should create successfully", func() { Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) }) From e95c033fe1a82b8e3b833531ec9bc10cabde961b Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Mon, 6 Apr 2026 19:49:07 +0300 Subject: [PATCH 064/102] handle defaults --- api/v1/perconaservermysql_types.go | 38 +++++++++++++++++--------- cmd/example-gen/pkg/defaults/manual.go | 12 ++++---- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index 616be8009..a8bb2048f 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -141,7 +141,7 @@ func (t ClusterType) isValid() bool { return false } -// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ''",message="mysql.image is required" +// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ”",message="mysql.image is required" // +kubebuilder:validation:XValidation:rule="has(self.size) && self.size > 0",message="mysql.size must be greater than 0" type MySQLSpec struct { // +kubebuilder:validation:Enum=group-replication;async @@ -178,7 +178,7 @@ type SidecarPVC struct { Spec corev1.PersistentVolumeClaimSpec `json:"spec"` } -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="orchestrator.image is required when orchestrator is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != ”)",message="orchestrator.image is required when orchestrator is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="orchestrator.size must be greater than 0 when orchestrator is enabled" type OrchestratorSpec struct { Enabled bool `json:"enabled,omitempty"` @@ -289,7 +289,7 @@ func (s *PodSpec) GetInitSpec(cr *PerconaServerMySQL) InitContainerSpec { return *s.InitContainer } -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="pmm.image is required when pmm is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != ”)",message="pmm.image is required when pmm is enabled" type PMMSpec struct { Enabled bool `json:"enabled,omitempty"` Image string `json:"image,omitempty"` @@ -518,7 +518,7 @@ func (b *BackupStorageAzureSpec) ContainerAndPrefix() (string, string) { } // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || has(self.binlogServer)",message="binlogServer is required when pitr is enabled" -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image != '')",message="binlogServer.image is required when pitr is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image != ”)",message="binlogServer.image is required when pitr is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size > 0)",message="binlogServer.size is required when pitr is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.serverId) && self.binlogServer.serverId > 0)",message="binlogServer.serverId is required when pitr is enabled" type PiTRSpec struct { @@ -545,15 +545,15 @@ type BinlogServerSpec struct { ServerID int32 `json:"serverId,omitempty"` // The number of seconds the utility will spend in disconnected mode between reconnection attempts. IdleTime int32 `json:"idleTime,omitempty"` - // SSLMode specifies the SSL mode for the connection to MySQL. Defaults to "verify_identity". + // SSLMode specifies the SSL mode for the connection to MySQL. SSLMode string `json:"sslMode,omitempty"` - // VerifyChecksum enables checksum verification during replication. Defaults to true. + // VerifyChecksum enables checksum verification during replication. VerifyChecksum *bool `json:"verifyChecksum,omitempty"` - // RewriteFileSize specifies the maximum binlog file size for rewrite. Defaults to "128M". + // RewriteFileSize specifies the maximum binlog file size for rewrite. RewriteFileSize string `json:"rewriteFileSize,omitempty"` - // CheckpointSize specifies the storage checkpoint size. Defaults to "2M". + // CheckpointSize specifies the storage checkpoint size. CheckpointSize string `json:"checkpointSize,omitempty"` - // CheckpointInterval specifies the storage checkpoint interval. Defaults to "30s". + // CheckpointInterval specifies the storage checkpoint interval. CheckpointInterval string `json:"checkpointInterval,omitempty"` PodSpec `json:",inline"` @@ -564,7 +564,7 @@ type ProxySpec struct { HAProxy *HAProxySpec `json:"haproxy,omitempty"` } -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="router.image is required when router is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != ”)",message="router.image is required when router is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="router.size must be greater than 0 when router is enabled" type MySQLRouterSpec struct { Enabled bool `json:"enabled,omitempty"` @@ -576,12 +576,12 @@ type MySQLRouterSpec struct { PodSpec `json:",inline"` } -// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ''",message="toolkit.image is required" +// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ”",message="toolkit.image is required" type ToolkitSpec struct { ContainerSpec `json:",inline"` } -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="haproxy.image is required when haproxy is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != ”)",message="haproxy.image is required when haproxy is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="haproxy.size must be greater than 0 when haproxy is enabled" type HAProxySpec struct { Enabled bool `json:"enabled,omitempty"` @@ -1052,11 +1052,23 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion bls.RewriteFileSize = "128M" } if bls.CheckpointSize == "" { - bls.CheckpointSize = "2M" + bls.CheckpointSize = "16M" } if bls.CheckpointInterval == "" { bls.CheckpointInterval = "30s" } + if bls.ConnectTimeout == 0 { + bls.ConnectTimeout = 30 + } + if bls.ReadTimeout == 0 { + bls.ReadTimeout = 30 + } + if bls.WriteTimeout == 0 { + bls.WriteTimeout = 30 + } + if bls.IdleTime == 0 { + bls.IdleTime = 30 + } } if cr.Spec.Pause { diff --git a/cmd/example-gen/pkg/defaults/manual.go b/cmd/example-gen/pkg/defaults/manual.go index 59c5a8108..bb80bebe0 100644 --- a/cmd/example-gen/pkg/defaults/manual.go +++ b/cmd/example-gen/pkg/defaults/manual.go @@ -133,11 +133,13 @@ func backupDefaults(spec *apiv1.BackupSpec) { EndpointURL: "https://s3.amazonaws.com", }, }, - ConnectTimeout: 10, - ReadTimeout: 10, - WriteTimeout: 10, - ServerID: 100, - IdleTime: 3, + ConnectTimeout: 30, + ReadTimeout: 30, + WriteTimeout: 30, + ServerID: 100, + IdleTime: 30, + CheckpointSize: "16M", + CheckpointInterval: "30s", }, } podSpecDefaults(&spec.PiTR.BinlogServer.PodSpec, ImageBinlogServer, corev1.ResourceRequirements{}, "", 30, nil, nil) From 24387dd5a4bcb79518f8cd6b7a7c83dac131980b Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 7 Apr 2026 11:53:07 +0300 Subject: [PATCH 065/102] run manifests --- .../bases/ps.percona.com_perconaservermysqls.yaml | 14 +++++++------- deploy/bundle.yaml | 14 +++++++------- deploy/cr.yaml | 12 ++++++------ deploy/crd.yaml | 14 +++++++------- deploy/cw-bundle.yaml | 14 +++++++------- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 50112cbcd..bcf6e5e43 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -1533,7 +1533,7 @@ spec: - message: binlogServer.image is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != '''')' + != ”)' - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -5731,7 +5731,7 @@ spec: type: object x-kubernetes-validations: - message: mysql.image is required - rule: has(self.image) && self.image != '' + rule: has(self.image) && self.image != ” - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -6978,7 +6978,7 @@ spec: x-kubernetes-validations: - message: orchestrator.image is required when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -7268,7 +7268,7 @@ spec: x-kubernetes-validations: - message: pmm.image is required when pmm is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' proxy: properties: haproxy: @@ -8515,7 +8515,7 @@ spec: x-kubernetes-validations: - message: haproxy.image is required when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -9789,7 +9789,7 @@ spec: x-kubernetes-validations: - message: router.image is required when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -10290,7 +10290,7 @@ spec: type: object x-kubernetes-validations: - message: toolkit.image is required - rule: has(self.image) && self.image != '' + rule: has(self.image) && self.image != ” unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 2d9654656..4b54d6af5 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -3896,7 +3896,7 @@ spec: - message: binlogServer.image is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != '''')' + != ”)' - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8094,7 +8094,7 @@ spec: type: object x-kubernetes-validations: - message: mysql.image is required - rule: has(self.image) && self.image != '' + rule: has(self.image) && self.image != ” - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9341,7 +9341,7 @@ spec: x-kubernetes-validations: - message: orchestrator.image is required when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9631,7 +9631,7 @@ spec: x-kubernetes-validations: - message: pmm.image is required when pmm is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' proxy: properties: haproxy: @@ -10878,7 +10878,7 @@ spec: x-kubernetes-validations: - message: haproxy.image is required when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12152,7 +12152,7 @@ spec: x-kubernetes-validations: - message: router.image is required when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12653,7 +12653,7 @@ spec: type: object x-kubernetes-validations: - message: toolkit.image is required - rule: has(self.image) && self.image != '' + rule: has(self.image) && self.image != ” unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/cr.yaml b/deploy/cr.yaml index aa2d0c169..4d57d7ae2 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -683,10 +683,10 @@ spec: # endpointUrl: https://s3.amazonaws.com # prefix: PREFIX_NAME # region: us-west-2 -# connectTimeout: 10 -# readTimeout: 10 -# writeTimeout: 10 -# idleTime: 3 +# connectTimeout: 30 +# readTimeout: 30 +# writeTimeout: 30 +# idleTime: 30 # affinity: # advanced: # nodeAffinity: @@ -699,8 +699,8 @@ spec: # - e2e-az1 # - e2e-az2 # antiAffinityTopologyKey: kubernetes.io/hostname -# checkpointInterval: "" -# checkpointSize: "" +# checkpointInterval: 30s +# checkpointSize: 16M # configuration: "" # containerSecurityContext: # privileged: false diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 6f974cddf..92f61300b 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -3896,7 +3896,7 @@ spec: - message: binlogServer.image is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != '''')' + != ”)' - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8094,7 +8094,7 @@ spec: type: object x-kubernetes-validations: - message: mysql.image is required - rule: has(self.image) && self.image != '' + rule: has(self.image) && self.image != ” - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9341,7 +9341,7 @@ spec: x-kubernetes-validations: - message: orchestrator.image is required when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9631,7 +9631,7 @@ spec: x-kubernetes-validations: - message: pmm.image is required when pmm is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' proxy: properties: haproxy: @@ -10878,7 +10878,7 @@ spec: x-kubernetes-validations: - message: haproxy.image is required when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12152,7 +12152,7 @@ spec: x-kubernetes-validations: - message: router.image is required when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12653,7 +12653,7 @@ spec: type: object x-kubernetes-validations: - message: toolkit.image is required - rule: has(self.image) && self.image != '' + rule: has(self.image) && self.image != ” unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index f31b1899e..5db7f66d0 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -3896,7 +3896,7 @@ spec: - message: binlogServer.image is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != '''')' + != ”)' - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8094,7 +8094,7 @@ spec: type: object x-kubernetes-validations: - message: mysql.image is required - rule: has(self.image) && self.image != '' + rule: has(self.image) && self.image != ” - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9341,7 +9341,7 @@ spec: x-kubernetes-validations: - message: orchestrator.image is required when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9631,7 +9631,7 @@ spec: x-kubernetes-validations: - message: pmm.image is required when pmm is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' proxy: properties: haproxy: @@ -10878,7 +10878,7 @@ spec: x-kubernetes-validations: - message: haproxy.image is required when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12152,7 +12152,7 @@ spec: x-kubernetes-validations: - message: router.image is required when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '''')' + && self.image != ”)' - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12653,7 +12653,7 @@ spec: type: object x-kubernetes-validations: - message: toolkit.image is required - rule: has(self.image) && self.image != '' + rule: has(self.image) && self.image != ” unsafeFlags: properties: backupNonReadyCluster: From 18d1a7bb6c56d9a73fc42cde34d63a2a58ae709c Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 7 Apr 2026 17:19:30 +0300 Subject: [PATCH 066/102] fixes on validations --- api/v1/perconaservermysql_types.go | 14 +++---- .../ps.percona.com_perconaservermysqls.yaml | 38 +++++++++---------- deploy/bundle.yaml | 38 +++++++++---------- deploy/crd.yaml | 38 +++++++++---------- deploy/cw-bundle.yaml | 38 +++++++++---------- 5 files changed, 83 insertions(+), 83 deletions(-) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index a8bb2048f..950240696 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -141,7 +141,7 @@ func (t ClusterType) isValid() bool { return false } -// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ”",message="mysql.image is required" +// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ''",message="mysql.image is required" // +kubebuilder:validation:XValidation:rule="has(self.size) && self.size > 0",message="mysql.size must be greater than 0" type MySQLSpec struct { // +kubebuilder:validation:Enum=group-replication;async @@ -178,7 +178,7 @@ type SidecarPVC struct { Spec corev1.PersistentVolumeClaimSpec `json:"spec"` } -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != ”)",message="orchestrator.image is required when orchestrator is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="orchestrator.image is required when orchestrator is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="orchestrator.size must be greater than 0 when orchestrator is enabled" type OrchestratorSpec struct { Enabled bool `json:"enabled,omitempty"` @@ -289,7 +289,7 @@ func (s *PodSpec) GetInitSpec(cr *PerconaServerMySQL) InitContainerSpec { return *s.InitContainer } -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != ”)",message="pmm.image is required when pmm is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="pmm.image is required when pmm is enabled" type PMMSpec struct { Enabled bool `json:"enabled,omitempty"` Image string `json:"image,omitempty"` @@ -518,7 +518,7 @@ func (b *BackupStorageAzureSpec) ContainerAndPrefix() (string, string) { } // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || has(self.binlogServer)",message="binlogServer is required when pitr is enabled" -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image != ”)",message="binlogServer.image is required when pitr is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image != '')",message="binlogServer.image is required when pitr is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size > 0)",message="binlogServer.size is required when pitr is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.serverId) && self.binlogServer.serverId > 0)",message="binlogServer.serverId is required when pitr is enabled" type PiTRSpec struct { @@ -564,7 +564,7 @@ type ProxySpec struct { HAProxy *HAProxySpec `json:"haproxy,omitempty"` } -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != ”)",message="router.image is required when router is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="router.image is required when router is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="router.size must be greater than 0 when router is enabled" type MySQLRouterSpec struct { Enabled bool `json:"enabled,omitempty"` @@ -576,12 +576,12 @@ type MySQLRouterSpec struct { PodSpec `json:",inline"` } -// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ”",message="toolkit.image is required" +// +kubebuilder:validation:XValidation:rule="has(self.image) && self.image != ''",message="toolkit.image is required" type ToolkitSpec struct { ContainerSpec `json:",inline"` } -// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != ”)",message="haproxy.image is required when haproxy is enabled" +// +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.image) && self.image != '')",message="haproxy.image is required when haproxy is enabled" // +kubebuilder:validation:XValidation:rule="!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)",message="haproxy.size must be greater than 0 when haproxy is enabled" type HAProxySpec struct { Enabled bool `json:"enabled,omitempty"` diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index bcf6e5e43..4d4e4bfed 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -1530,10 +1530,10 @@ spec: x-kubernetes-validations: - message: binlogServer is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' - - message: binlogServer.image is required when pitr is enabled - rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + - message: "binlogServer.image is required when pitr is enabled" + rule: "!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != ”)' + != '')" - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -5730,8 +5730,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: mysql.image is required - rule: has(self.image) && self.image != ” + - message: "mysql.image is required" + rule: "has(self.image) && self.image != ''" - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -6976,9 +6976,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: orchestrator.image is required when orchestrator is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "orchestrator.image is required when orchestrator is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -7266,9 +7266,9 @@ spec: type: string type: object x-kubernetes-validations: - - message: pmm.image is required when pmm is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "pmm.image is required when pmm is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" proxy: properties: haproxy: @@ -8513,9 +8513,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: haproxy.image is required when haproxy is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "haproxy.image is required when haproxy is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -9787,9 +9787,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: router.image is required when router is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "router.image is required when router is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -10289,8 +10289,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: toolkit.image is required - rule: has(self.image) && self.image != ” + - message: "toolkit.image is required" + rule: "has(self.image) && self.image != ''" unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 4b54d6af5..ec5631718 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -3893,10 +3893,10 @@ spec: x-kubernetes-validations: - message: binlogServer is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' - - message: binlogServer.image is required when pitr is enabled - rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + - message: "binlogServer.image is required when pitr is enabled" + rule: "!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != ”)' + != '')" - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8093,8 +8093,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: mysql.image is required - rule: has(self.image) && self.image != ” + - message: "mysql.image is required" + rule: "has(self.image) && self.image != ''" - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9339,9 +9339,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: orchestrator.image is required when orchestrator is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "orchestrator.image is required when orchestrator is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9629,9 +9629,9 @@ spec: type: string type: object x-kubernetes-validations: - - message: pmm.image is required when pmm is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "pmm.image is required when pmm is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" proxy: properties: haproxy: @@ -10876,9 +10876,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: haproxy.image is required when haproxy is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "haproxy.image is required when haproxy is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12150,9 +12150,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: router.image is required when router is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "router.image is required when router is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12652,8 +12652,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: toolkit.image is required - rule: has(self.image) && self.image != ” + - message: "toolkit.image is required" + rule: "has(self.image) && self.image != ''" unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 92f61300b..62c6cf481 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -3893,10 +3893,10 @@ spec: x-kubernetes-validations: - message: binlogServer is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' - - message: binlogServer.image is required when pitr is enabled - rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + - message: "binlogServer.image is required when pitr is enabled" + rule: "!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != ”)' + != '')" - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8093,8 +8093,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: mysql.image is required - rule: has(self.image) && self.image != ” + - message: "mysql.image is required" + rule: "has(self.image) && self.image != ''" - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9339,9 +9339,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: orchestrator.image is required when orchestrator is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "orchestrator.image is required when orchestrator is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9629,9 +9629,9 @@ spec: type: string type: object x-kubernetes-validations: - - message: pmm.image is required when pmm is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "pmm.image is required when pmm is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" proxy: properties: haproxy: @@ -10876,9 +10876,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: haproxy.image is required when haproxy is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "haproxy.image is required when haproxy is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12150,9 +12150,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: router.image is required when router is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "router.image is required when router is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12652,8 +12652,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: toolkit.image is required - rule: has(self.image) && self.image != ” + - message: "toolkit.image is required" + rule: "has(self.image) && self.image != ''" unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 5db7f66d0..e877ad515 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -3893,10 +3893,10 @@ spec: x-kubernetes-validations: - message: binlogServer is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' - - message: binlogServer.image is required when pitr is enabled - rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + - message: "binlogServer.image is required when pitr is enabled" + rule: "!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != ”)' + != '')" - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8093,8 +8093,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: mysql.image is required - rule: has(self.image) && self.image != ” + - message: "mysql.image is required" + rule: "has(self.image) && self.image != ''" - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9339,9 +9339,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: orchestrator.image is required when orchestrator is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "orchestrator.image is required when orchestrator is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9629,9 +9629,9 @@ spec: type: string type: object x-kubernetes-validations: - - message: pmm.image is required when pmm is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "pmm.image is required when pmm is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" proxy: properties: haproxy: @@ -10876,9 +10876,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: haproxy.image is required when haproxy is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "haproxy.image is required when haproxy is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12150,9 +12150,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: router.image is required when router is enabled - rule: '!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != ”)' + - message: "router.image is required when router is enabled" + rule: "!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '')" - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12652,8 +12652,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: toolkit.image is required - rule: has(self.image) && self.image != ” + - message: "toolkit.image is required" + rule: "has(self.image) && self.image != ''" unsafeFlags: properties: backupNonReadyCluster: From a7c2052ebf8f6972fbef4ece590de7eb9c4545a2 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 7 Apr 2026 17:23:06 +0300 Subject: [PATCH 067/102] run manifests --- .../ps.percona.com_perconaservermysqls.yaml | 38 +++++++++---------- deploy/bundle.yaml | 38 +++++++++---------- deploy/crd.yaml | 38 +++++++++---------- deploy/cw-bundle.yaml | 38 +++++++++---------- 4 files changed, 76 insertions(+), 76 deletions(-) diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 4d4e4bfed..50112cbcd 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -1530,10 +1530,10 @@ spec: x-kubernetes-validations: - message: binlogServer is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' - - message: "binlogServer.image is required when pitr is enabled" - rule: "!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + - message: binlogServer.image is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != '')" + != '''')' - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -5730,8 +5730,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: "mysql.image is required" - rule: "has(self.image) && self.image != ''" + - message: mysql.image is required + rule: has(self.image) && self.image != '' - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -6976,9 +6976,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "orchestrator.image is required when orchestrator is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: orchestrator.image is required when orchestrator is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -7266,9 +7266,9 @@ spec: type: string type: object x-kubernetes-validations: - - message: "pmm.image is required when pmm is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: pmm.image is required when pmm is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' proxy: properties: haproxy: @@ -8513,9 +8513,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "haproxy.image is required when haproxy is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: haproxy.image is required when haproxy is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -9787,9 +9787,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "router.image is required when router is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: router.image is required when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -10289,8 +10289,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: "toolkit.image is required" - rule: "has(self.image) && self.image != ''" + - message: toolkit.image is required + rule: has(self.image) && self.image != '' unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index ec5631718..2d9654656 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -3893,10 +3893,10 @@ spec: x-kubernetes-validations: - message: binlogServer is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' - - message: "binlogServer.image is required when pitr is enabled" - rule: "!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + - message: binlogServer.image is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != '')" + != '''')' - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8093,8 +8093,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: "mysql.image is required" - rule: "has(self.image) && self.image != ''" + - message: mysql.image is required + rule: has(self.image) && self.image != '' - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9339,9 +9339,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "orchestrator.image is required when orchestrator is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: orchestrator.image is required when orchestrator is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9629,9 +9629,9 @@ spec: type: string type: object x-kubernetes-validations: - - message: "pmm.image is required when pmm is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: pmm.image is required when pmm is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' proxy: properties: haproxy: @@ -10876,9 +10876,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "haproxy.image is required when haproxy is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: haproxy.image is required when haproxy is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12150,9 +12150,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "router.image is required when router is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: router.image is required when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12652,8 +12652,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: "toolkit.image is required" - rule: "has(self.image) && self.image != ''" + - message: toolkit.image is required + rule: has(self.image) && self.image != '' unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 62c6cf481..6f974cddf 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -3893,10 +3893,10 @@ spec: x-kubernetes-validations: - message: binlogServer is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' - - message: "binlogServer.image is required when pitr is enabled" - rule: "!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + - message: binlogServer.image is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != '')" + != '''')' - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8093,8 +8093,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: "mysql.image is required" - rule: "has(self.image) && self.image != ''" + - message: mysql.image is required + rule: has(self.image) && self.image != '' - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9339,9 +9339,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "orchestrator.image is required when orchestrator is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: orchestrator.image is required when orchestrator is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9629,9 +9629,9 @@ spec: type: string type: object x-kubernetes-validations: - - message: "pmm.image is required when pmm is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: pmm.image is required when pmm is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' proxy: properties: haproxy: @@ -10876,9 +10876,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "haproxy.image is required when haproxy is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: haproxy.image is required when haproxy is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12150,9 +12150,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "router.image is required when router is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: router.image is required when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12652,8 +12652,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: "toolkit.image is required" - rule: "has(self.image) && self.image != ''" + - message: toolkit.image is required + rule: has(self.image) && self.image != '' unsafeFlags: properties: backupNonReadyCluster: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index e877ad515..f31b1899e 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -3893,10 +3893,10 @@ spec: x-kubernetes-validations: - message: binlogServer is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || has(self.binlogServer)' - - message: "binlogServer.image is required when pitr is enabled" - rule: "!(has(self.enabled) && self.enabled) || !has(self.binlogServer) + - message: binlogServer.image is required when pitr is enabled + rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.image) && self.binlogServer.image - != '')" + != '''')' - message: binlogServer.size is required when pitr is enabled rule: '!(has(self.enabled) && self.enabled) || !has(self.binlogServer) || (has(self.binlogServer.size) && self.binlogServer.size @@ -8093,8 +8093,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: "mysql.image is required" - rule: "has(self.image) && self.image != ''" + - message: mysql.image is required + rule: has(self.image) && self.image != '' - message: mysql.size must be greater than 0 rule: has(self.size) && self.size > 0 orchestrator: @@ -9339,9 +9339,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "orchestrator.image is required when orchestrator is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: orchestrator.image is required when orchestrator is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: orchestrator.size must be greater than 0 when orchestrator is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && @@ -9629,9 +9629,9 @@ spec: type: string type: object x-kubernetes-validations: - - message: "pmm.image is required when pmm is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: pmm.image is required when pmm is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' proxy: properties: haproxy: @@ -10876,9 +10876,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "haproxy.image is required when haproxy is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: haproxy.image is required when haproxy is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: haproxy.size must be greater than 0 when haproxy is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) @@ -12150,9 +12150,9 @@ spec: type: array type: object x-kubernetes-validations: - - message: "router.image is required when router is enabled" - rule: "!(has(self.enabled) && self.enabled) || (has(self.image) - && self.image != '')" + - message: router.image is required when router is enabled + rule: '!(has(self.enabled) && self.enabled) || (has(self.image) + && self.image != '''')' - message: router.size must be greater than 0 when router is enabled rule: '!(has(self.enabled) && self.enabled) || (has(self.size) && self.size > 0)' @@ -12652,8 +12652,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: "toolkit.image is required" - rule: "has(self.image) && self.image != ''" + - message: toolkit.image is required + rule: has(self.image) && self.image != '' unsafeFlags: properties: backupNonReadyCluster: From 88562d7d4b4b317acbaf7f3ff7aff74c02841c7e Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Wed, 8 Apr 2026 00:05:01 +0300 Subject: [PATCH 068/102] fix unit test for required server id --- pkg/controller/ps/controller_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/ps/controller_test.go b/pkg/controller/ps/controller_test.go index b643fb9b6..dd93cae59 100644 --- a/pkg/controller/ps/controller_test.go +++ b/pkg/controller/ps/controller_test.go @@ -2158,6 +2158,7 @@ var _ = Describe("BinlogServer", Ordered, func() { CredentialsSecret: "s3-secret", }, }, + ServerID: 1, PodSpec: psv1.PodSpec{ Size: 1, ContainerSpec: psv1.ContainerSpec{ From 9b7561cb0ff2cf9ccf8c0d7a44906f2542954eee Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Wed, 8 Apr 2026 12:21:08 +0300 Subject: [PATCH 069/102] fix unit test for defaults --- api/v1/perconaservermysql_types_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1/perconaservermysql_types_test.go b/api/v1/perconaservermysql_types_test.go index b628d48fd..0203cedf5 100644 --- a/api/v1/perconaservermysql_types_test.go +++ b/api/v1/perconaservermysql_types_test.go @@ -191,7 +191,7 @@ func TestCheckNSetDefaults(t *testing.T) { assert.NotNil(t, bls.VerifyChecksum) assert.True(t, *bls.VerifyChecksum) assert.Equal(t, "128M", bls.RewriteFileSize) - assert.Equal(t, "2M", bls.CheckpointSize) + assert.Equal(t, "16M", bls.CheckpointSize) assert.Equal(t, "30s", bls.CheckpointInterval) }) t.Run("binlog server explicit values are not overridden by defaults", func(t *testing.T) { From cc4625e8ff7b9292d268ee92704356f5f2b1bb4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 9 Apr 2026 09:11:30 +0300 Subject: [PATCH 070/102] sleep-forever --- build/run-pitr-restore.sh | 9 +++++++++ pkg/pitr/pitr.go | 11 +++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh index dfae2c189..9cf29581f 100644 --- a/build/run-pitr-restore.sh +++ b/build/run-pitr-restore.sh @@ -18,6 +18,15 @@ until mysqladmin -u operator -p$( Date: Thu, 9 Apr 2026 09:11:46 +0300 Subject: [PATCH 071/102] use SQL_AFTER_GTIDS --- cmd/internal/db/db.go | 2 +- cmd/pitr/main.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/internal/db/db.go b/cmd/internal/db/db.go index ad8470069..0ebe47e0f 100644 --- a/cmd/internal/db/db.go +++ b/cmd/internal/db/db.go @@ -409,7 +409,7 @@ func (d *DB) ChangeReplicationSourceRelay(ctx context.Context, relayLogFile stri func (d *DB) StartReplicaUntilGTID(ctx context.Context, gtid string) error { _, err := d.db.ExecContext(ctx, fmt.Sprintf( - "START REPLICA SQL_THREAD UNTIL SQL_BEFORE_GTIDS='%s'", gtid)) + "START REPLICA SQL_THREAD UNTIL SQL_AFTER_GTIDS='%s'", gtid)) return errors.Wrap(err, "start replica until GTID") } diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 534554e5c..bf4c3f1b9 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -205,12 +205,12 @@ func runApply(ctx context.Context, newDB newDatabaseFn, getSecret func(apiv1.Sys } log.Printf("GTID_EXECUTED: %s", currentGTID) - log.Println("running 'CHANGE REPLICATION SOURCE'") + log.Printf("CHANGE REPLICATION SOURCE TO RELAY_LOG_FILE='%s', RELAY_LOG_POS=%d, SOURCE_HOST='dummy'", firstRelayLog, 4) if err := database.ChangeReplicationSourceRelay(ctx, firstRelayLog, 4); err != nil { return fmt.Errorf("change replication source: %w", err) } - log.Printf("starting replica until GTID: %s", pitrGTID) + log.Printf("START REPLICA SQL_THREAD UNTIL SQL_AFTER_GTIDS='%s'", pitrGTID) if err := database.StartReplicaUntilGTID(ctx, pitrGTID); err != nil { return fmt.Errorf("start replica until GTID: %w", err) } From f780ffdd38663028969e3eb6719276b02e117b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Fri, 10 Apr 2026 16:03:07 +0300 Subject: [PATCH 072/102] implement pipe based recovery --- build/run-pitr-restore.sh | 6 +- cmd/pitr/main.go | 267 +++++++++++-------------- cmd/pitr/main_test.go | 405 +++++++++++++------------------------- 3 files changed, 254 insertions(+), 424 deletions(-) diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh index 9cf29581f..58588154a 100644 --- a/build/run-pitr-restore.sh +++ b/build/run-pitr-restore.sh @@ -7,10 +7,7 @@ set -e echo "Starting mysqld" mysqld \ --admin-address=127.0.0.1 \ - --skip-replica-start \ --user=mysql \ - --read-only=ON \ - --super-read-only=ON \ --gtid-mode=ON \ --enforce-gtid-consistency=ON >/tmp/mysqld.log 2>&1 & @@ -27,8 +24,7 @@ if [[ -n ${SLEEP_FOREVER} ]]; then exit 0 fi -/opt/percona/pitr setup -/opt/percona/pitr apply +/opt/percona/pitr echo "Stopping mysqld" mysqladmin -u operator -p$(/dev/null diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index bf4c3f1b9..7441a8ab7 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" @@ -11,24 +12,16 @@ import ( "os/exec" "path/filepath" "strings" - "time" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" "github.com/percona/percona-server-mysql-operator/cmd/bootstrap/utils" "github.com/percona/percona-server-mysql-operator/cmd/internal/db" "github.com/percona/percona-server-mysql-operator/pkg/binlogserver" "github.com/percona/percona-server-mysql-operator/pkg/xtrabackup/storage" - "github.com/pkg/errors" ) -// Database defines the MySQL operations needed for PITR apply. +// Database defines the MySQL operations needed for PITR. type Database interface { - ChangeReplicationSourceRelay(ctx context.Context, relayLogFile string, relayLogPos int) error - StartReplicaUntilGTID(ctx context.Context, gtid string) error - WaitReplicaSQLThreadStop(ctx context.Context, pollInterval time.Duration) error - StopReplication(ctx context.Context) error - ResetReplication(ctx context.Context) error - SetGTIDNextAutomatic(ctx context.Context) error GetGTIDExecuted(ctx context.Context) (string, error) Close() error } @@ -36,31 +29,23 @@ type Database interface { type newStorageFn func(ctx context.Context, endpoint, accessKey, secretKey, bucket, prefix, region string, verifyTLS bool) (storage.Storage, error) type newDatabaseFn func(ctx context.Context, params db.DBParams) (Database, error) -func main() { - if len(os.Args) < 2 { - log.Fatal("usage: pitr ") - } +// applyBinlogsFn starts a single mysql client process and for each binlog file +// runs mysqlbinlog with the given args, piping the output into mysql's stdin. +type applyBinlogsFn func(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string) error +func main() { ctx := context.Background() - switch os.Args[1] { - case "setup": - if err := runSetup(ctx, storage.NewS3, "/var/lib/mysql"); err != nil { - log.Fatalf("setup failed: %v", err) - } - case "apply": - newDB := func(ctx context.Context, params db.DBParams) (Database, error) { - return db.NewDatabase(ctx, params) - } - if err := runApply(ctx, newDB, utils.GetSecret, getLatestGTIDByDatetime, "/var/lib/mysql"); err != nil { - log.Fatalf("apply failed: %v", err) - } - default: - log.Fatalf("unknown subcommand: %s", os.Args[1]) + newDB := func(ctx context.Context, params db.DBParams) (Database, error) { + return db.NewDatabase(ctx, params) + } + + if err := run(ctx, storage.NewS3, newDB, utils.GetSecret, applyBinlogs); err != nil { + log.Fatalf("pitr failed: %v", err) } } -func runSetup(ctx context.Context, newS3 newStorageFn, mysqlDir string) error { +func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret func(apiv1.SystemUser) (string, error), apply applyBinlogsFn) error { binlogsPath := os.Getenv("BINLOGS_PATH") if binlogsPath == "" { return fmt.Errorf("BINLOGS_PATH is not set") @@ -80,11 +65,34 @@ func runSetup(ctx context.Context, newS3 newStorageFn, mysqlDir string) error { return fmt.Errorf("no binlog entries found") } - hostname, err := os.Hostname() + pitrType := os.Getenv("PITR_TYPE") + pitrDate := os.Getenv("PITR_DATE") + pitrGTID := os.Getenv("PITR_GTID") + + // Connect to MySQL to get the backup's GTID_EXECUTED. + operatorPass, err := getSecret(apiv1.UserOperator) if err != nil { - return fmt.Errorf("get hostname: %w", err) + return fmt.Errorf("get operator password: %w", err) } + database, err := newDB(ctx, db.DBParams{ + User: apiv1.UserOperator, + Pass: operatorPass, + Host: "127.0.0.1", + }) + if err != nil { + return fmt.Errorf("connect to MySQL: %w", err) + } + + gtidExecuted, err := database.GetGTIDExecuted(ctx) + if err != nil { + database.Close() + return fmt.Errorf("get GTID_EXECUTED: %w", err) + } + log.Printf("GTID_EXECUTED from backup: %s", gtidExecuted) + database.Close() + + // Create S3 client and download binlogs on the fly. endpoint := os.Getenv("AWS_ENDPOINT") accessKey := os.Getenv("AWS_ACCESS_KEY_ID") secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") @@ -97,184 +105,135 @@ func runSetup(ctx context.Context, newS3 newStorageFn, mysqlDir string) error { return fmt.Errorf("create S3 client: %w", err) } - var relayLogFiles []string + tmpDir, err := os.MkdirTemp("", "pitr-binlogs-") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + var binlogPaths []string for i, entry := range entries { - relayLogName := fmt.Sprintf("%s-relay-bin.%06d", hostname, i+1) - relayLogPath := filepath.Join(mysqlDir, relayLogName) + binlogName := fmt.Sprintf("binlog.%06d", i+1) + binlogPath := filepath.Join(tmpDir, binlogName) objectKey, err := objectKeyFromURI(entry.URI, bucket) if err != nil { return fmt.Errorf("parse URI %s: %w", entry.URI, err) } - log.Printf("downloading binlog %s to %s", objectKey, relayLogPath) + log.Printf("downloading binlog %s to %s", objectKey, binlogPath) obj, err := s3Client.GetObject(ctx, objectKey) if err != nil { return fmt.Errorf("download binlog %s: %w", entry.URI, err) } - f, err := os.Create(relayLogPath) + f, err := os.Create(binlogPath) if err != nil { - if closeErr := obj.Close(); closeErr != nil { - log.Printf("close object %s: %v", entry.URI, closeErr) - } - return fmt.Errorf("create relay log file %s: %w", relayLogPath, err) + obj.Close() + return fmt.Errorf("create binlog file %s: %w", binlogPath, err) } _, err = io.Copy(f, obj) - if closeErr := obj.Close(); closeErr != nil { - log.Printf("close object %s: %v", entry.URI, closeErr) - } - if closeErr := f.Close(); closeErr != nil { - log.Printf("close relay log file %s: %v", relayLogPath, closeErr) - } + obj.Close() + f.Close() if err != nil { - return fmt.Errorf("write relay log file %s: %w", relayLogPath, err) + return fmt.Errorf("write binlog file %s: %w", binlogPath, err) } - relayLogFiles = append(relayLogFiles, "./"+relayLogName) + binlogPaths = append(binlogPaths, binlogPath) } - indexPath := filepath.Join(mysqlDir, fmt.Sprintf("%s-relay-bin.index", hostname)) - indexContent := strings.Join(relayLogFiles, "\n") + "\n" - if err := os.WriteFile(indexPath, []byte(indexContent), 0644); err != nil { - return fmt.Errorf("write relay log index: %w", err) + // Build mysqlbinlog args. + mysqlbinlogArgs := []string{"--disable-log-bin"} + if gtidExecuted != "" { + mysqlbinlogArgs = append(mysqlbinlogArgs, fmt.Sprintf("--exclude-gtids=%s", gtidExecuted)) } - log.Printf("setup complete: %d relay log files written", len(relayLogFiles)) - return nil -} - -func runApply(ctx context.Context, newDB newDatabaseFn, getSecret func(apiv1.SystemUser) (string, error), getGTIDByDatetime func(string, string) (string, error), mysqlDir string) error { - pitrType := os.Getenv("PITR_TYPE") - pitrDate := os.Getenv("PITR_DATE") - pitrGTID := os.Getenv("PITR_GTID") + switch pitrType { + case "date": + mysqlbinlogArgs = append(mysqlbinlogArgs, fmt.Sprintf("--stop-datetime=%s", pitrDate)) + case "gtid": + mysqlbinlogArgs = append(mysqlbinlogArgs, fmt.Sprintf("--include-gtids=%s", pitrGTID)) + default: + return fmt.Errorf("unknown PITR_TYPE: %s", pitrType) + } - hostname, err := os.Hostname() - if err != nil { - return fmt.Errorf("get hostname: %w", err) + // Build mysql client args. + mysqlArgs := []string{ + "-u", string(apiv1.UserOperator), + fmt.Sprintf("-p%s", operatorPass), + "-h", "127.0.0.1", + "-P", "33062", } - operatorPass, err := getSecret(apiv1.UserOperator) - if err != nil { - return fmt.Errorf("get operator password: %w", err) + log.Printf("applying %d binlog(s) with mysqlbinlog args: %v", len(binlogPaths), mysqlbinlogArgs) + + if err := apply(ctx, binlogPaths, mysqlbinlogArgs, mysqlArgs); err != nil { + return fmt.Errorf("apply binlogs: %w", err) } - database, err := newDB(ctx, db.DBParams{ + // Reconnect to log the final GTID state. + database, err = newDB(ctx, db.DBParams{ User: apiv1.UserOperator, Pass: operatorPass, Host: "127.0.0.1", }) if err != nil { - return fmt.Errorf("connect to MySQL: %w", err) + return fmt.Errorf("reconnect to MySQL: %w", err) } - defer func() { - if err := database.Close(); err != nil { - log.Printf("close database: %v", err) - } - }() + defer database.Close() - binlogsPath := os.Getenv("BINLOGS_PATH") - data, err := os.ReadFile(binlogsPath) + gtidExecuted, err = database.GetGTIDExecuted(ctx) if err != nil { - return fmt.Errorf("read binlogs file: %w", err) - } - - var entries []binlogserver.BinlogEntry - if err := json.Unmarshal(data, &entries); err != nil { - return fmt.Errorf("parse binlogs json: %w", err) - } - - lastRelayLog := fmt.Sprintf("%s-relay-bin.%06d", hostname, len(entries)) - lastRelayLogPath := filepath.Join(mysqlDir, lastRelayLog) - - if pitrType == "date" { - pitrGTID, err = getGTIDByDatetime(lastRelayLogPath, pitrDate) - if err != nil { - return fmt.Errorf("get latest GTID for date %s: %w", pitrDate, err) - } - log.Printf("latest GTID for date %s: %s", pitrDate, pitrGTID) + return fmt.Errorf("get GTID_EXECUTED after restore: %w", err) } + log.Printf("GTID_EXECUTED after PITR: %s", gtidExecuted) - firstRelayLog := fmt.Sprintf("%s-relay-bin.000001", hostname) + log.Println("PITR complete") + return nil +} - currentGTID, err := database.GetGTIDExecuted(ctx) +// applyBinlogs starts a single mysql client and for each binlog file +// spawns mysqlbinlog, piping its output into mysql's stdin. +func applyBinlogs(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string) error { + mysqlCmd := exec.CommandContext(ctx, "mysql", mysqlArgs...) + mysqlStdin, err := mysqlCmd.StdinPipe() if err != nil { - return fmt.Errorf("get current GTID_EXECUTED: %w", err) + return fmt.Errorf("create mysql stdin pipe: %w", err) } - log.Printf("GTID_EXECUTED: %s", currentGTID) - log.Printf("CHANGE REPLICATION SOURCE TO RELAY_LOG_FILE='%s', RELAY_LOG_POS=%d, SOURCE_HOST='dummy'", firstRelayLog, 4) - if err := database.ChangeReplicationSourceRelay(ctx, firstRelayLog, 4); err != nil { - return fmt.Errorf("change replication source: %w", err) - } + var mysqlStderr bytes.Buffer + mysqlCmd.Stderr = &mysqlStderr - log.Printf("START REPLICA SQL_THREAD UNTIL SQL_AFTER_GTIDS='%s'", pitrGTID) - if err := database.StartReplicaUntilGTID(ctx, pitrGTID); err != nil { - return fmt.Errorf("start replica until GTID: %w", err) + if err := mysqlCmd.Start(); err != nil { + return fmt.Errorf("start mysql: %w", err) } - log.Println("waiting for replication to complete...") - if err := database.WaitReplicaSQLThreadStop(ctx, time.Second); err != nil { - return fmt.Errorf("wait for replication: %w", err) - } + for _, binlogPath := range binlogPaths { + args := append(mysqlbinlogArgs, binlogPath) + binlogCmd := exec.CommandContext(ctx, "mysqlbinlog", args...) - log.Println("stopping replication") - if err := database.StopReplication(ctx); err != nil { - return errors.Wrap(err, "stop replication") - } + var binlogStderr bytes.Buffer + binlogCmd.Stdout = mysqlStdin + binlogCmd.Stderr = &binlogStderr - log.Println("running 'RESET REPLICA ALL'") - if err := database.ResetReplication(ctx); err != nil { - return errors.Wrap(err, "reset replication") - } + log.Printf("running: mysqlbinlog %s", binlogPath) - currentGTID, err = database.GetGTIDExecuted(ctx) - if err != nil { - return fmt.Errorf("get GTID_EXECUTED after restore: %w", err) - } - log.Printf("GTID_EXECUTED: %s", currentGTID) - - log.Println("setting GTID_NEXT to AUTOMATIC") - if err := database.SetGTIDNextAutomatic(ctx); err != nil { - return fmt.Errorf("set GTID_NEXT to AUTOMATIC: %w", err) - } - - log.Println("PITR apply complete") - return nil -} - -func getLatestGTIDByDatetime(relayLogPath, startDatetime string) (string, error) { - t, err := time.ParseInLocation("2006-01-02 15:04:05", startDatetime, time.UTC) - if err != nil { - return "", fmt.Errorf("parse datetime %q: %w", startDatetime, err) - } - stopDatetime := t.Add(time.Second).Format("2006-01-02 15:04:05") - - cmd := exec.Command("bash", "-c", - fmt.Sprintf("mysqlbinlog --stop-datetime='%s' %s | grep GTID_NEXT | grep -v AUTOMATIC | tail -n 1", - stopDatetime, relayLogPath)) - - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("failed to execute mysqlbinlog pipeline: %w", err) + if err := binlogCmd.Run(); err != nil { + mysqlStdin.Close() + mysqlCmd.Wait() + return fmt.Errorf("mysqlbinlog %s failed: %w, stderr: %s", binlogPath, err, binlogStderr.String()) + } } - line := strings.TrimSpace(string(output)) - if line == "" { - return "", fmt.Errorf("no GTID found at %s in %s", startDatetime, relayLogPath) - } + mysqlStdin.Close() - // Extract GTID from: SET @@SESSION.GTID_NEXT= 'uuid:n,uuid:n'/*!*/; - start := strings.Index(line, "'") - end := strings.LastIndex(line, "'") - if start == -1 || end == -1 || start == end { - return "", fmt.Errorf("failed to parse GTID from line: %s", line) + if err := mysqlCmd.Wait(); err != nil { + return fmt.Errorf("mysql failed: %w, stderr: %s", err, mysqlStderr.String()) } - gtid := line[start+1 : end] - return gtid, nil + return nil } // objectKeyFromURI extracts the S3 object key from a full URI. diff --git a/cmd/pitr/main_test.go b/cmd/pitr/main_test.go index 80020f2b7..65a4fbf05 100644 --- a/cmd/pitr/main_test.go +++ b/cmd/pitr/main_test.go @@ -4,13 +4,10 @@ import ( "context" "encoding/json" "errors" - "fmt" "io" "os" - "path/filepath" "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -43,49 +40,10 @@ func (f *fakeStorage) DeleteObject(_ context.Context, _ string) error func (f *fakeStorage) SetPrefix(_ string) {} func (f *fakeStorage) GetPrefix() string { return "" } -// fakeDB records method calls and returns configured errors. type fakeDB struct { - changeRelayErr error - startUntilErr error - waitErr error - stopErr error - resetErr error - setGTIDNextErr error - getGTIDExecutedErr error getGTIDExecutedResult string + getGTIDExecutedErr error calls []string - startUntilGTID string -} - -func (f *fakeDB) ChangeReplicationSourceRelay(_ context.Context, _ string, _ int) error { - f.calls = append(f.calls, "ChangeReplicationSourceRelay") - return f.changeRelayErr -} - -func (f *fakeDB) StartReplicaUntilGTID(_ context.Context, gtid string) error { - f.calls = append(f.calls, "StartReplicaUntilGTID") - f.startUntilGTID = gtid - return f.startUntilErr -} - -func (f *fakeDB) WaitReplicaSQLThreadStop(_ context.Context, _ time.Duration) error { - f.calls = append(f.calls, "WaitReplicaSQLThreadStop") - return f.waitErr -} - -func (f *fakeDB) StopReplication(_ context.Context) error { - f.calls = append(f.calls, "StopReplication") - return f.stopErr -} - -func (f *fakeDB) ResetReplication(_ context.Context) error { - f.calls = append(f.calls, "ResetReplication") - return f.resetErr -} - -func (f *fakeDB) SetGTIDNextAutomatic(_ context.Context) error { - f.calls = append(f.calls, "SetGTIDNextAutomatic") - return f.setGTIDNextErr } func (f *fakeDB) GetGTIDExecuted(_ context.Context) (string, error) { @@ -107,45 +65,83 @@ func writeBinlogsFile(t *testing.T, entries []binlogserver.BinlogEntry) string { return f.Name() } -func TestRunSetup(t *testing.T) { +type applyCall struct { + binlogPaths []string + mysqlbinlogArgs []string + mysqlArgs []string +} + +func TestRun(t *testing.T) { bucket := "mybucket" + defaultEntries := []binlogserver.BinlogEntry{ + {URI: "s3://mybucket/binlogs/binlog.000001"}, + {URI: "s3://mybucket/binlogs/binlog.000002"}, + } + + defaultS3 := func(fake *fakeStorage) newStorageFn { + fake.objects = map[string]string{ + "binlogs/binlog.000001": "binlogdata1", + "binlogs/binlog.000002": "binlogdata2", + } + return func(_ context.Context, _, _, _, _, _, _ string, _ bool) (storage.Storage, error) { + return fake, nil + } + } + tests := map[string]struct { - setupEnv func(t *testing.T, binlogsPath string) entries []binlogserver.BinlogEntry rawContent string + pitrType string + pitrGTID string + pitrDate string + db *fakeDB + newDB func(ctx context.Context, params db.DBParams) (Database, error) newS3 func(*fakeStorage) newStorageFn + getSecret func(apiv1.SystemUser) (string, error) + applyErr error expectedError string - checkResult func(t *testing.T, mysqlDir string) + checkApply func(t *testing.T, call applyCall) }{ "missing BINLOGS_PATH": { - setupEnv: func(t *testing.T, _ string) { - t.Setenv("BINLOGS_PATH", "") - }, expectedError: "BINLOGS_PATH", }, "invalid JSON in binlogs file": { - setupEnv: func(t *testing.T, binlogsPath string) { - t.Setenv("BINLOGS_PATH", binlogsPath) - }, rawContent: "not-json", expectedError: "parse binlogs json", }, "empty binlog entries": { - setupEnv: func(t *testing.T, binlogsPath string) { - t.Setenv("BINLOGS_PATH", binlogsPath) - }, entries: []binlogserver.BinlogEntry{}, expectedError: "no binlog entries found", }, - "S3 client creation error": { - setupEnv: func(t *testing.T, binlogsPath string) { - t.Setenv("BINLOGS_PATH", binlogsPath) - t.Setenv("S3_BUCKET", bucket) - }, - entries: []binlogserver.BinlogEntry{ - {URI: "s3://mybucket/binlogs/binlog.000001"}, + "get secret error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + getSecret: func(apiv1.SystemUser) (string, error) { return "", errors.New("secret not found") }, + expectedError: "get operator password", + }, + "DB connect error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + newDB: func(_ context.Context, _ db.DBParams) (Database, error) { + return nil, errors.New("connection refused") }, + expectedError: "connect to MySQL", + }, + "GetGTIDExecuted error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{getGTIDExecutedErr: errors.New("query failed")}, + expectedError: "get GTID_EXECUTED", + }, + "S3 client creation error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{getGTIDExecutedResult: "uuid:1-5"}, newS3: func(_ *fakeStorage) newStorageFn { return func(_ context.Context, _, _, _, _, _, _ string, _ bool) (storage.Storage, error) { return nil, errors.New("s3 unavailable") @@ -153,14 +149,11 @@ func TestRunSetup(t *testing.T) { }, expectedError: "create S3 client", }, - "GetObject error": { - setupEnv: func(t *testing.T, binlogsPath string) { - t.Setenv("BINLOGS_PATH", binlogsPath) - t.Setenv("S3_BUCKET", bucket) - }, - entries: []binlogserver.BinlogEntry{ - {URI: "s3://mybucket/binlogs/binlog.000001"}, - }, + "S3 download error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1", + db: &fakeDB{getGTIDExecutedResult: "uuid:1-5"}, newS3: func(fake *fakeStorage) newStorageFn { fake.getErr = errors.New("download failed") return func(_ context.Context, _, _, _, _, _, _ string, _ bool) (storage.Storage, error) { @@ -169,41 +162,60 @@ func TestRunSetup(t *testing.T) { }, expectedError: "download binlog", }, - "success": { - setupEnv: func(t *testing.T, binlogsPath string) { - t.Setenv("BINLOGS_PATH", binlogsPath) - t.Setenv("S3_BUCKET", bucket) - }, - entries: []binlogserver.BinlogEntry{ - {URI: "s3://mybucket/binlogs/binlog.000001"}, - {URI: "s3://mybucket/binlogs/binlog.000002"}, + "unknown PITR type": { + entries: defaultEntries, + pitrType: "unknown", + db: &fakeDB{getGTIDExecutedResult: "uuid:1-5"}, + expectedError: "unknown PITR_TYPE", + }, + "apply error": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "uuid:1-10", + db: &fakeDB{getGTIDExecutedResult: "uuid:1-5"}, + applyErr: errors.New("mysql failed"), + expectedError: "apply binlogs", + }, + "GTID mode success": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "aaaaaaaa-0000-0000-0000-000000000001:1-10", + db: &fakeDB{getGTIDExecutedResult: "aaaaaaaa-0000-0000-0000-000000000001:1-5"}, + checkApply: func(t *testing.T, call applyCall) { + assert.Len(t, call.binlogPaths, 2) + assert.Contains(t, call.mysqlbinlogArgs, "--disable-log-bin") + assert.Contains(t, call.mysqlbinlogArgs, "--exclude-gtids=aaaaaaaa-0000-0000-0000-000000000001:1-5") + assert.Contains(t, call.mysqlbinlogArgs, "--include-gtids=aaaaaaaa-0000-0000-0000-000000000001:1-10") + assert.NotContains(t, call.mysqlbinlogArgs, "--stop-datetime") }, - newS3: func(fake *fakeStorage) newStorageFn { - fake.objects = map[string]string{ - "binlogs/binlog.000001": "binlogdata1", - "binlogs/binlog.000002": "binlogdata2", - } - return func(_ context.Context, _, _, _, _, _, _ string, _ bool) (storage.Storage, error) { - return fake, nil + }, + "date mode success": { + entries: defaultEntries, + pitrType: "date", + pitrDate: "2024-01-15 12:00:00", + db: &fakeDB{getGTIDExecutedResult: "bbbbbbbb-0000-0000-0000-000000000002:1-5"}, + checkApply: func(t *testing.T, call applyCall) { + assert.Len(t, call.binlogPaths, 2) + assert.Contains(t, call.mysqlbinlogArgs, "--disable-log-bin") + assert.Contains(t, call.mysqlbinlogArgs, "--exclude-gtids=bbbbbbbb-0000-0000-0000-000000000002:1-5") + assert.Contains(t, call.mysqlbinlogArgs, "--stop-datetime=2024-01-15 12:00:00") + // Should not contain --include-gtids for date mode + for _, arg := range call.mysqlbinlogArgs { + assert.False(t, strings.HasPrefix(arg, "--include-gtids"), "date mode should not have --include-gtids") } }, - checkResult: func(t *testing.T, mysqlDir string) { - hostname, err := os.Hostname() - require.NoError(t, err) - - indexPath := filepath.Join(mysqlDir, hostname+"-relay-bin.index") - indexData, err := os.ReadFile(indexPath) - require.NoError(t, err, "relay log index file must exist") - - indexContent := string(indexData) - assert.Contains(t, indexContent, hostname+"-relay-bin.000001") - assert.Contains(t, indexContent, hostname+"-relay-bin.000002") - - for i, wantContent := range []string{"binlogdata1", "binlogdata2"} { - relayLog := filepath.Join(mysqlDir, fmt.Sprintf("%s-relay-bin.%06d", hostname, i+1)) - data, err := os.ReadFile(relayLog) - require.NoErrorf(t, err, "relay log %d must exist", i+1) - assert.Equalf(t, wantContent, string(data), "relay log %d content mismatch", i+1) + }, + "empty GTID_EXECUTED": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "aaaaaaaa-0000-0000-0000-000000000001:1-10", + db: &fakeDB{getGTIDExecutedResult: ""}, + checkApply: func(t *testing.T, call applyCall) { + assert.Contains(t, call.mysqlbinlogArgs, "--disable-log-bin") + assert.Contains(t, call.mysqlbinlogArgs, "--include-gtids=aaaaaaaa-0000-0000-0000-000000000001:1-10") + // No --exclude-gtids when GTID_EXECUTED is empty + for _, arg := range call.mysqlbinlogArgs { + assert.False(t, strings.HasPrefix(arg, "--exclude-gtids"), "should not have --exclude-gtids when GTID_EXECUTED is empty") } }, }, @@ -211,6 +223,7 @@ func TestRunSetup(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { + // Set up binlogs file. var binlogsPath string if tc.rawContent != "" { f, err := os.CreateTemp(t.TempDir(), "binlogs-*.json") @@ -223,159 +236,15 @@ func TestRunSetup(t *testing.T) { binlogsPath = writeBinlogsFile(t, tc.entries) } - tc.setupEnv(t, binlogsPath) - - fake := &fakeStorage{} - var newS3 newStorageFn - if tc.newS3 != nil { - newS3 = tc.newS3(fake) - } - - mysqlDir := t.TempDir() - err := runSetup(t.Context(), newS3, mysqlDir) - - if tc.expectedError != "" { - require.ErrorContains(t, err, tc.expectedError) - return - } - require.NoError(t, err) - if tc.checkResult != nil { - tc.checkResult(t, mysqlDir) + if binlogsPath != "" { + t.Setenv("BINLOGS_PATH", binlogsPath) + } else { + t.Setenv("BINLOGS_PATH", "") } - }) - } -} - -func TestRunApply(t *testing.T) { - defaultEntries := []binlogserver.BinlogEntry{ - {URI: "s3://bucket/binlogs/binlog.000001"}, - {URI: "s3://bucket/binlogs/binlog.000002"}, - } - allDBCalls := []string{ - "GetGTIDExecuted", - "ChangeReplicationSourceRelay", - "StartReplicaUntilGTID", - "WaitReplicaSQLThreadStop", - "StopReplication", - "ResetReplication", - "GetGTIDExecuted", - "SetGTIDNextAutomatic", - } - - tests := map[string]struct { - entries []binlogserver.BinlogEntry - pitrType string - pitrGTID string - pitrDate string - db *fakeDB - newDB func(ctx context.Context, params db.DBParams) (Database, error) - getSecret func(apiv1.SystemUser) (string, error) - getGTID func(string, string) (string, error) - expectedError string - expectedFuncCalls []string - expectedUDID string - }{ - "get secret error": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "uuid:1", - getSecret: func(apiv1.SystemUser) (string, error) { return "", errors.New("secret not found") }, - expectedError: "get operator password", - }, - "DB connect error": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "uuid:1", - newDB: func(_ context.Context, _ db.DBParams) (Database, error) { - return nil, errors.New("connection refused") - }, - expectedError: "connect to MySQL", - }, - "change replication source relay error": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "uuid:1", - db: &fakeDB{changeRelayErr: errors.New("relay error")}, - expectedError: "change replication source", - expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay"}, - }, - "start replica until GTID error": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "uuid:1", - db: &fakeDB{startUntilErr: errors.New("start error")}, - expectedError: "start replica until GTID", - expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay", "StartReplicaUntilGTID"}, - }, - "wait replica stop error": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "uuid:1", - db: &fakeDB{waitErr: errors.New("wait error")}, - expectedError: "wait for replication", - expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay", "StartReplicaUntilGTID", "WaitReplicaSQLThreadStop"}, - }, - "stop replication error": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "uuid:1", - db: &fakeDB{stopErr: errors.New("stop error")}, - expectedError: "stop replication", - expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay", "StartReplicaUntilGTID", "WaitReplicaSQLThreadStop", "StopReplication"}, - }, - "reset replication error": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "uuid:1", - db: &fakeDB{resetErr: errors.New("reset error")}, - expectedError: "reset replication", - expectedFuncCalls: []string{"GetGTIDExecuted", "ChangeReplicationSourceRelay", "StartReplicaUntilGTID", "WaitReplicaSQLThreadStop", "StopReplication", "ResetReplication"}, - }, - "set GTID_NEXT error": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "uuid:1", - db: &fakeDB{setGTIDNextErr: errors.New("gtid error")}, - expectedError: "set GTID_NEXT to AUTOMATIC", - expectedFuncCalls: allDBCalls, - }, - "GTID mode success": { - entries: defaultEntries, - pitrType: "gtid", - pitrGTID: "aaaaaaaa-0000-0000-0000-000000000001:1-10", - db: &fakeDB{}, - expectedFuncCalls: allDBCalls, - expectedUDID: "aaaaaaaa-0000-0000-0000-000000000001:1-10", - }, - "date mode success": { - entries: defaultEntries, - pitrType: "date", - pitrDate: "2024-01-15 12:00:00", - db: &fakeDB{}, - getGTID: func(_ string, datetime string) (string, error) { - assert.Equal(t, "2024-01-15 12:00:00", datetime) - return "bbbbbbbb-0000-0000-0000-000000000002:1-5", nil - }, - expectedFuncCalls: allDBCalls, - expectedUDID: "bbbbbbbb-0000-0000-0000-000000000002:1-5", - }, - "date mode getGTID error": { - entries: defaultEntries, - pitrType: "date", - pitrDate: "2024-01-15 12:00:00", - db: &fakeDB{}, - getGTID: func(string, string) (string, error) { return "", errors.New("mysqlbinlog failed") }, - expectedError: "get latest GTID for date", - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - binlogsPath := writeBinlogsFile(t, tc.entries) - t.Setenv("BINLOGS_PATH", binlogsPath) t.Setenv("PITR_TYPE", tc.pitrType) t.Setenv("PITR_GTID", tc.pitrGTID) t.Setenv("PITR_DATE", tc.pitrDate) + t.Setenv("S3_BUCKET", bucket) fakeDatabase := tc.db @@ -391,27 +260,33 @@ func TestRunApply(t *testing.T) { getSecret = func(apiv1.SystemUser) (string, error) { return "testpass", nil } } - getGTID := tc.getGTID - if getGTID == nil { - getGTID = func(string, string) (string, error) { - t.Fatal("getGTIDByDatetime should not be called") - return "", nil + fake := &fakeStorage{} + var newS3 newStorageFn + if tc.newS3 != nil { + newS3 = tc.newS3(fake) + } else { + newS3 = defaultS3(fake) + } + + var captured applyCall + apply := func(_ context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string) error { + captured = applyCall{ + binlogPaths: binlogPaths, + mysqlbinlogArgs: mysqlbinlogArgs, + mysqlArgs: mysqlArgs, } + return tc.applyErr } - err := runApply(t.Context(), newDB, getSecret, getGTID, t.TempDir()) + err := run(t.Context(), newS3, newDB, getSecret, apply) if tc.expectedError != "" { require.ErrorContains(t, err, tc.expectedError) - } else { - require.NoError(t, err) - } - - if tc.expectedFuncCalls != nil && fakeDatabase != nil { - assert.Equal(t, tc.expectedFuncCalls, fakeDatabase.calls) + return } - if tc.expectedUDID != "" && fakeDatabase != nil { - assert.Equal(t, tc.expectedUDID, fakeDatabase.startUntilGTID) + require.NoError(t, err) + if tc.checkApply != nil { + tc.checkApply(t, captured) } }) } From df264a3b7549bfaa90f7793c1da1cbda1008b024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Fri, 10 Apr 2026 16:03:17 +0300 Subject: [PATCH 073/102] fix e2e test --- e2e-tests/tests/gr-pitr-minio/07-assert.yaml | 6 +- .../tests/gr-pitr-minio/10-failover.yaml | 4 +- .../gr-pitr-minio/12-write-and-restore.yaml | 4 +- .../tests/gr-pitr-minio/14-switchover.yaml | 64 ------------------- 4 files changed, 7 insertions(+), 71 deletions(-) delete mode 100644 e2e-tests/tests/gr-pitr-minio/14-switchover.yaml diff --git a/e2e-tests/tests/gr-pitr-minio/07-assert.yaml b/e2e-tests/tests/gr-pitr-minio/07-assert.yaml index bbd6bf1cc..c2d3d8132 100644 --- a/e2e-tests/tests/gr-pitr-minio/07-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/07-assert.yaml @@ -7,18 +7,18 @@ apiVersion: v1 metadata: name: 07-read-data-0 data: - max_id: "100501" + max_id: "100502" --- kind: ConfigMap apiVersion: v1 metadata: name: 07-read-data-1 data: - max_id: "100501" + max_id: "100502" --- kind: ConfigMap apiVersion: v1 metadata: name: 07-read-data-2 data: - max_id: "100501" + max_id: "100502" diff --git a/e2e-tests/tests/gr-pitr-minio/10-failover.yaml b/e2e-tests/tests/gr-pitr-minio/10-failover.yaml index 7f8404385..b5ac51568 100644 --- a/e2e-tests/tests/gr-pitr-minio/10-failover.yaml +++ b/e2e-tests/tests/gr-pitr-minio/10-failover.yaml @@ -1,8 +1,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 600 commands: - - script: |- + - timeout: 600 + script: |- set -o errexit set -o xtrace diff --git a/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml b/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml index 556a565c3..382e032c1 100644 --- a/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml +++ b/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml @@ -1,8 +1,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 600 commands: - - script: |- + - timeout: 600 + script: |- set -o errexit set -o xtrace diff --git a/e2e-tests/tests/gr-pitr-minio/14-switchover.yaml b/e2e-tests/tests/gr-pitr-minio/14-switchover.yaml deleted file mode 100644 index 899a70d31..000000000 --- a/e2e-tests/tests/gr-pitr-minio/14-switchover.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -timeout: 300 -commands: - - script: |- - set -o errexit - set -o xtrace - - source ../../functions - - cluster_name=$(get_cluster_name) - primary_before=$(get_primary_from_label) - - new_primary="" - for i in 0 1 2; do - pod="${cluster_name}-mysql-${i}" - if [[ "${pod}" != "${primary_before}" ]]; then - new_primary="${pod}" - break - fi - done - - if [[ -z "${new_primary}" ]]; then - echo "Could not find a secondary pod to switchover to" - exit 1 - fi - - new_primary_fqdn="${new_primary}.${cluster_name}-mysql.${NAMESPACE}:3306" - echo "Switching primary from ${primary_before} to ${new_primary} (${new_primary_fqdn})" - - uri=$(get_mysqlsh_uri_for_pod "${primary_before}") - client_pod=$(get_client_pod) - wait_pod "${client_pod}" 1>&2 - - kubectl -n "${NAMESPACE}" exec "${client_pod}" -- \ - mysqlsh --js --quiet-start=2 --uri "${uri}" -- cluster setPrimary "${new_primary_fqdn}" - - wait_cluster_consistency_gr "${cluster_name}" 3 3 - - primary_after=$(get_primary_from_label) - - if [[ "${primary_before}" == "${primary_after}" ]]; then - echo "Primary pod did not change after switchover: was ${primary_before}, still ${primary_after}" - exit 1 - fi - - if [[ "${primary_after}" != "${new_primary}" ]]; then - echo "Primary after switchover (${primary_after}) does not match expected (${new_primary})" - exit 1 - fi - echo "Primary changed from ${primary_before} to ${primary_after}" - - retry=0 - until [[ "$(kubectl -n "${NAMESPACE}" get endpoints "${cluster_name}-primary" \ - -o jsonpath='{.subsets[0].addresses[0].targetRef.name}' 2>/dev/null)" == "${primary_after}" ]]; do - sleep 5 - retry=$((retry + 1)) - if [ $retry -ge 24 ]; then - echo "Primary service endpoint did not update to ${primary_after} after 2 minutes" - kubectl -n "${NAMESPACE}" get endpoints "${cluster_name}-primary" -o yaml - exit 1 - fi - done - echo "Primary service endpoint correctly points to new primary after switchover: ${primary_after}" From 9c06f7a4a10a7761e8a9acf4f0628d586a5ed0db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Sun, 12 Apr 2026 21:27:47 +0300 Subject: [PATCH 074/102] fix timeout --- e2e-tests/tests/gr-pitr-minio/06-assert.yaml | 4 ++++ e2e-tests/tests/gr-pitr-minio/12-assert.yaml | 4 ++++ .../tests/gr-pitr-minio/99-remove-cluster-gracefully.yaml | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/e2e-tests/tests/gr-pitr-minio/06-assert.yaml b/e2e-tests/tests/gr-pitr-minio/06-assert.yaml index b50e013c5..841aefa9e 100644 --- a/e2e-tests/tests/gr-pitr-minio/06-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/06-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 +collectors: + - type: pod + selector: "app.kubernetes.io/component=pitr" + tail: 1000 --- apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL diff --git a/e2e-tests/tests/gr-pitr-minio/12-assert.yaml b/e2e-tests/tests/gr-pitr-minio/12-assert.yaml index bebfc90f6..f51749937 100644 --- a/e2e-tests/tests/gr-pitr-minio/12-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/12-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 +collectors: + - type: pod + selector: "app.kubernetes.io/component=pitr" + tail: 1000 --- apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL diff --git a/e2e-tests/tests/gr-pitr-minio/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/gr-pitr-minio/99-remove-cluster-gracefully.yaml index 35b44e567..5f3cc2863 100644 --- a/e2e-tests/tests/gr-pitr-minio/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/gr-pitr-minio/99-remove-cluster-gracefully.yaml @@ -13,4 +13,4 @@ commands: source ../../functions destroy_operator - timeout: 60 + timeout: 180 From eb9951f6ab187aae63423a77ba7a017813bee441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 13 Apr 2026 17:51:42 +0300 Subject: [PATCH 075/102] fix tls-cert-manager --- e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml | 3 +++ e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml | 3 +++ e2e-tests/tests/tls-cert-manager/03-check-cert.yaml | 3 +++ e2e-tests/tests/tls-cert-manager/05-check-cert.yaml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml b/e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml index 7a44e42ab..c9ec13821 100644 --- a/e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml +++ b/e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml @@ -12,6 +12,9 @@ commands: "*.gr-tls-cert-manager-mysql", "*.gr-tls-cert-manager-mysql.'"${NAMESPACE}"'", "*.gr-tls-cert-manager-mysql.'"${NAMESPACE}"'.svc", + "tls-cert-manager-mysql-primary", + "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'", + "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'.svc", "*.gr-tls-cert-manager-orchestrator", "*.gr-tls-cert-manager-orchestrator.'"${NAMESPACE}"'", "*.gr-tls-cert-manager-orchestrator.'"${NAMESPACE}"'.svc", diff --git a/e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml b/e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml index 35ca29679..84e4ea121 100644 --- a/e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml +++ b/e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml @@ -12,6 +12,9 @@ commands: "*.gr-tls-cert-manager-mysql", "*.gr-tls-cert-manager-mysql.'"${NAMESPACE}"'", "*.gr-tls-cert-manager-mysql.'"${NAMESPACE}"'.svc", + "tls-cert-manager-mysql-primary", + "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'", + "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'.svc", "*.gr-tls-cert-manager-orchestrator", "*.gr-tls-cert-manager-orchestrator.'"${NAMESPACE}"'", "*.gr-tls-cert-manager-orchestrator.'"${NAMESPACE}"'.svc", diff --git a/e2e-tests/tests/tls-cert-manager/03-check-cert.yaml b/e2e-tests/tests/tls-cert-manager/03-check-cert.yaml index b65c07694..909f92d02 100644 --- a/e2e-tests/tests/tls-cert-manager/03-check-cert.yaml +++ b/e2e-tests/tests/tls-cert-manager/03-check-cert.yaml @@ -12,6 +12,9 @@ commands: "*.tls-cert-manager-mysql", "*.tls-cert-manager-mysql.'"${NAMESPACE}"'", "*.tls-cert-manager-mysql.'"${NAMESPACE}"'.svc", + "tls-cert-manager-mysql-primary", + "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'", + "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'.svc", "*.tls-cert-manager-orchestrator", "*.tls-cert-manager-orchestrator.'"${NAMESPACE}"'", "*.tls-cert-manager-orchestrator.'"${NAMESPACE}"'.svc", diff --git a/e2e-tests/tests/tls-cert-manager/05-check-cert.yaml b/e2e-tests/tests/tls-cert-manager/05-check-cert.yaml index d1b5b786c..3a44eac24 100644 --- a/e2e-tests/tests/tls-cert-manager/05-check-cert.yaml +++ b/e2e-tests/tests/tls-cert-manager/05-check-cert.yaml @@ -12,6 +12,9 @@ commands: "*.tls-cert-manager-mysql", "*.tls-cert-manager-mysql.'"${NAMESPACE}"'", "*.tls-cert-manager-mysql.'"${NAMESPACE}"'.svc", + "tls-cert-manager-mysql-primary", + "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'", + "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'.svc", "*.tls-cert-manager-orchestrator", "*.tls-cert-manager-orchestrator.'"${NAMESPACE}"'", "*.tls-cert-manager-orchestrator.'"${NAMESPACE}"'.svc", From 4fd8584d5a98b5772f6387f674c36badf18a8b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 13 Apr 2026 18:06:38 +0300 Subject: [PATCH 076/102] address copilot comments --- build/run-pitr-restore.sh | 5 +++-- pkg/controller/psrestore/controller.go | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh index 58588154a..96b1dcd16 100644 --- a/build/run-pitr-restore.sh +++ b/build/run-pitr-restore.sh @@ -11,7 +11,8 @@ mysqld \ --gtid-mode=ON \ --enforce-gtid-consistency=ON >/tmp/mysqld.log 2>&1 & -until mysqladmin -u operator -p$(/dev/null; do +echo "Waiting for mysqld to be ready" +until mysqladmin -u operator -p"$(/dev/null; do sleep 1; done @@ -27,4 +28,4 @@ fi /opt/percona/pitr echo "Stopping mysqld" -mysqladmin -u operator -p$(/dev/null +mysqladmin -u operator -p"$(/dev/null diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index abc27c855..82e5a8242 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -181,6 +181,12 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req defer r.sm.Delete(cr.Spec.ClusterName) if cr.Spec.PITR != nil { + if !cluster.Spec.Backup.PiTR.Enabled || cluster.Spec.Backup.PiTR.BinlogServer == nil { + status.State = apiv1.RestoreError + status.StateDesc = "PITR is not enabled for the cluster" + return ctrl.Result{}, nil + } + status.State = apiv1.RestoreStarting if err := r.reconcilePITRConfig(ctx, cr, cluster); err != nil { status.State = apiv1.RestoreError From b642ed1c0acb98572eed6a0f83e7d8e0826c7347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 13 Apr 2026 18:11:16 +0300 Subject: [PATCH 077/102] fix upgrade tests --- .../tests/async-upgrade/01-create-cluster.yaml | 14 ++++++++++++++ e2e-tests/tests/gr-upgrade/01-create-cluster.yaml | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/e2e-tests/tests/async-upgrade/01-create-cluster.yaml b/e2e-tests/tests/async-upgrade/01-create-cluster.yaml index 195a7c86a..bd94986a5 100644 --- a/e2e-tests/tests/async-upgrade/01-create-cluster.yaml +++ b/e2e-tests/tests/async-upgrade/01-create-cluster.yaml @@ -15,9 +15,23 @@ commands: exit 1 fi + # binlog server options were required in v1.0.0 + # we can delete them after v1.1.0 released get_cr_with_latest_versions_in_vs \ | yq eval "$(printf '.spec.initImage="%s"' "${init_image}")" - \ | yq eval "$(printf '.spec.crVersion="%s"' "${version}")" - \ | yq eval '.spec.mysql.clusterType="async"' - \ | yq eval '.spec.secretsName="async-upgrade-secrets"' - \ + | yq eval '.spec.backup.pitr.enabled=false' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.prefix="binlogs"' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.pitr.binlogServer.storage.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.pitr.binlogServer.size=1' - \ + | yq eval '.spec.backup.pitr.binlogServer.serverId=100' - \ + | yq eval '.spec.backup.pitr.binlogServer.connectTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.readTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.writeTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.idleTime=3' - \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/gr-upgrade/01-create-cluster.yaml b/e2e-tests/tests/gr-upgrade/01-create-cluster.yaml index f35d312c0..fd603a0dc 100644 --- a/e2e-tests/tests/gr-upgrade/01-create-cluster.yaml +++ b/e2e-tests/tests/gr-upgrade/01-create-cluster.yaml @@ -15,6 +15,8 @@ commands: exit 1 fi + # binlog server options were required in v1.0.0 + # we can delete them after v1.1.0 released get_cr_with_latest_versions_in_vs \ | yq eval "$(printf '.spec.initImage="%s"' "${init_image}")" - \ | yq eval "$(printf '.spec.crVersion="%s"' "${version}")" - \ @@ -22,4 +24,16 @@ commands: | yq eval '.spec.mysql.clusterType="group-replication"' - \ | yq eval '.spec.proxy.router.enabled=false' - \ | yq eval '.spec.proxy.haproxy.enabled=true' - \ + | yq eval '.spec.backup.pitr.enabled=false' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.prefix="binlogs"' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.pitr.binlogServer.storage.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.pitr.binlogServer.size=1' - \ + | yq eval '.spec.backup.pitr.binlogServer.serverId=100' - \ + | yq eval '.spec.backup.pitr.binlogServer.connectTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.readTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.writeTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.idleTime=3' - \ | kubectl -n "${NAMESPACE}" apply -f - From ecb632afccf7b9ab633fff84c49a4ed72cbff685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 13 Apr 2026 18:11:27 +0300 Subject: [PATCH 078/102] use new image for PBS --- deploy/cr.yaml | 2 +- e2e-tests/vars.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 4d57d7ae2..83ffef485 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -670,7 +670,7 @@ spec: enabled: false # binlogServer: # size: 1 -# image: perconalab/percona-binlog-server:0.2.0 +# image: perconalab/percona-binlog-server:0.2.1 # imagePullPolicy: Always # imagePullSecrets: # - name: my-secret-1 diff --git a/e2e-tests/vars.sh b/e2e-tests/vars.sh index 6577d98e0..50b2e9d63 100755 --- a/e2e-tests/vars.sh +++ b/e2e-tests/vars.sh @@ -22,7 +22,7 @@ export IMAGE_ORCHESTRATOR=${IMAGE_ORCHESTRATOR:-"perconalab/percona-server-mysql export IMAGE_ROUTER=${IMAGE_ROUTER:-"perconalab/percona-server-mysql-operator:main-router${MYSQL_VERSION}"} export IMAGE_TOOLKIT=${IMAGE_TOOLKIT:-"perconalab/percona-server-mysql-operator:main-toolkit"} export IMAGE_HAPROXY=${IMAGE_HAPROXY:-"perconalab/percona-server-mysql-operator:main-haproxy"} -export IMAGE_BINLOG_SERVER=${IMAGE_BINLOG_SERVER:-"perconalab/percona-binlog-server:0.2.0"} +export IMAGE_BINLOG_SERVER=${IMAGE_BINLOG_SERVER:-"perconalab/percona-binlog-server:0.2.1"} export PMM_SERVER_VERSION=${PMM_SERVER_VERSION:-"1.4.3"} export IMAGE_PMM_CLIENT=${IMAGE_PMM_CLIENT:-"perconalab/pmm-client:3-dev-latest"} export IMAGE_PMM_SERVER=${IMAGE_PMM_SERVER:-"perconalab/pmm-server:3-dev-latest"} From b7997fcc0716cc81853f8db4eb73e51025cf626e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 13 Apr 2026 22:45:28 +0300 Subject: [PATCH 079/102] fix --- pkg/controller/psrestore/controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index 82e5a8242..10cc08d6b 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -181,9 +181,9 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req defer r.sm.Delete(cr.Spec.ClusterName) if cr.Spec.PITR != nil { - if !cluster.Spec.Backup.PiTR.Enabled || cluster.Spec.Backup.PiTR.BinlogServer == nil { + if cluster.Spec.Backup.PiTR.BinlogServer == nil { status.State = apiv1.RestoreError - status.StateDesc = "PITR is not enabled for the cluster" + status.StateDesc = "Binlog server is not enabled for the cluster" return ctrl.Result{}, nil } From 2a10e7bd76a67ab3e12074ff49e398137592a659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 14 Apr 2026 14:35:41 +0300 Subject: [PATCH 080/102] check for pitr enabled before restore --- api/v1/perconaservermysql_types.go | 4 +++- pkg/controller/psrestore/controller.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index 950240696..2e1844507 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -1076,7 +1076,9 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion cr.Spec.Orchestrator.Size = 0 cr.Spec.Proxy.Router.Size = 0 cr.Spec.Proxy.HAProxy.Size = 0 - cr.Spec.Backup.PiTR.Enabled = false + if cr.Spec.Backup.PiTR.BinlogServer != nil { + cr.Spec.Backup.PiTR.BinlogServer.Size = 0 + } } if cr.Spec.SecretsName == "" { diff --git a/pkg/controller/psrestore/controller.go b/pkg/controller/psrestore/controller.go index 10cc08d6b..5d76de8f7 100644 --- a/pkg/controller/psrestore/controller.go +++ b/pkg/controller/psrestore/controller.go @@ -181,7 +181,7 @@ func (r *PerconaServerMySQLRestoreReconciler) Reconcile(ctx context.Context, req defer r.sm.Delete(cr.Spec.ClusterName) if cr.Spec.PITR != nil { - if cluster.Spec.Backup.PiTR.BinlogServer == nil { + if !cluster.Spec.Backup.PiTR.Enabled || cluster.Spec.Backup.PiTR.BinlogServer == nil { status.State = apiv1.RestoreError status.StateDesc = "Binlog server is not enabled for the cluster" return ctrl.Result{}, nil From c59fb23e87e8a1d5e0801049bc35ddb97810fad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 14 Apr 2026 14:35:58 +0300 Subject: [PATCH 081/102] remove unused code --- cmd/internal/db/db.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cmd/internal/db/db.go b/cmd/internal/db/db.go index 0ebe47e0f..d985ef02a 100644 --- a/cmd/internal/db/db.go +++ b/cmd/internal/db/db.go @@ -400,19 +400,6 @@ func (d *DB) ResetBinaryLogAndGTIDs(ctx context.Context) error { return errors.Wrap(err, "reset binary logs and gtids") } -func (d *DB) ChangeReplicationSourceRelay(ctx context.Context, relayLogFile string, relayLogPos int) error { - _, err := d.db.ExecContext(ctx, fmt.Sprintf( - "CHANGE REPLICATION SOURCE TO RELAY_LOG_FILE='%s', RELAY_LOG_POS=%d, SOURCE_HOST='dummy'", - relayLogFile, relayLogPos)) - return errors.Wrap(err, "change replication source to relay log") -} - -func (d *DB) StartReplicaUntilGTID(ctx context.Context, gtid string) error { - _, err := d.db.ExecContext(ctx, fmt.Sprintf( - "START REPLICA SQL_THREAD UNTIL SQL_AFTER_GTIDS='%s'", gtid)) - return errors.Wrap(err, "start replica until GTID") -} - func (d *DB) StartReplicaUntilPosition(ctx context.Context, relayLogFile string, relayLogPos int) error { _, err := d.db.ExecContext(ctx, fmt.Sprintf( "START REPLICA SQL_THREAD UNTIL RELAY_LOG_FILE='%s', RELAY_LOG_POS=%d", From 11d251d2e47ed2a653d06c395f63628c38430e30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 14 Apr 2026 14:36:07 +0300 Subject: [PATCH 082/102] fix default value --- cmd/example-gen/pkg/defaults/values.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/example-gen/pkg/defaults/values.go b/cmd/example-gen/pkg/defaults/values.go index 44205bf96..d9ea30703 100644 --- a/cmd/example-gen/pkg/defaults/values.go +++ b/cmd/example-gen/pkg/defaults/values.go @@ -16,7 +16,7 @@ const ( ImageOrchestrator = "perconalab/percona-server-mysql-operator:main-orchestrator" ImagePMM = "perconalab/pmm-client:3-dev-latest" ImageBackup = "perconalab/percona-server-mysql-operator:main-backup8.4" - ImageBinlogServer = "perconalab/percona-binlog-server:0.2.0" + ImageBinlogServer = "perconalab/percona-binlog-server:0.2.1" ImageToolkit = "perconalab/percona-server-mysql-operator:main-toolkit" ) From bcf66076e1fceda8f278d582e668686b65f5d508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 14 Apr 2026 15:19:29 +0300 Subject: [PATCH 083/102] use MYSQL_PWD env var --- cmd/pitr/main.go | 9 ++++----- cmd/pitr/main_test.go | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 7441a8ab7..858699411 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -31,7 +31,7 @@ type newDatabaseFn func(ctx context.Context, params db.DBParams) (Database, erro // applyBinlogsFn starts a single mysql client process and for each binlog file // runs mysqlbinlog with the given args, piping the output into mysql's stdin. -type applyBinlogsFn func(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string) error +type applyBinlogsFn func(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string, mysqlPass string) error func main() { ctx := context.Background() @@ -92,7 +92,6 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret log.Printf("GTID_EXECUTED from backup: %s", gtidExecuted) database.Close() - // Create S3 client and download binlogs on the fly. endpoint := os.Getenv("AWS_ENDPOINT") accessKey := os.Getenv("AWS_ACCESS_KEY_ID") secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") @@ -162,14 +161,13 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret // Build mysql client args. mysqlArgs := []string{ "-u", string(apiv1.UserOperator), - fmt.Sprintf("-p%s", operatorPass), "-h", "127.0.0.1", "-P", "33062", } log.Printf("applying %d binlog(s) with mysqlbinlog args: %v", len(binlogPaths), mysqlbinlogArgs) - if err := apply(ctx, binlogPaths, mysqlbinlogArgs, mysqlArgs); err != nil { + if err := apply(ctx, binlogPaths, mysqlbinlogArgs, mysqlArgs, operatorPass); err != nil { return fmt.Errorf("apply binlogs: %w", err) } @@ -196,8 +194,9 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret // applyBinlogs starts a single mysql client and for each binlog file // spawns mysqlbinlog, piping its output into mysql's stdin. -func applyBinlogs(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string) error { +func applyBinlogs(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string, mysqlPass string) error { mysqlCmd := exec.CommandContext(ctx, "mysql", mysqlArgs...) + mysqlCmd.Env = append(os.Environ(), fmt.Sprintf("MYSQL_PWD=%s", mysqlPass)) mysqlStdin, err := mysqlCmd.StdinPipe() if err != nil { return fmt.Errorf("create mysql stdin pipe: %w", err) diff --git a/cmd/pitr/main_test.go b/cmd/pitr/main_test.go index 65a4fbf05..63f5e54ec 100644 --- a/cmd/pitr/main_test.go +++ b/cmd/pitr/main_test.go @@ -269,7 +269,7 @@ func TestRun(t *testing.T) { } var captured applyCall - apply := func(_ context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string) error { + apply := func(_ context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string, _ string) error { captured = applyCall{ binlogPaths: binlogPaths, mysqlbinlogArgs: mysqlbinlogArgs, From 32d7307f0e83c10fd2a0bd4c5cff73b13df77315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 14 Apr 2026 15:25:43 +0300 Subject: [PATCH 084/102] improve parseEndpointUrl --- pkg/controller/ps/controller.go | 10 ---- pkg/controller/ps/endpoint.go | 19 +++++++ pkg/controller/ps/endpoint_test.go | 86 ++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 pkg/controller/ps/endpoint.go create mode 100644 pkg/controller/ps/endpoint_test.go diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index b8040a82d..cb08f9b7e 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -1413,16 +1413,6 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context return nil } -// parseEndpointURL extracts the protocol and host from an endpoint URL. -// Expected formats: "s3://s3.amazonaws.com", "https://minio-service:9000" -func parseEndpointURL(endpointURL string) (protocol, host string, err error) { - idx := strings.Index(endpointURL, "://") - if idx < 0 { - return "", "", fmt.Errorf("endpoint URL %q must include protocol (e.g. s3://... or https://...)", endpointURL) - } - return endpointURL[:idx], endpointURL[idx+3:], nil -} - func (r *PerconaServerMySQLReconciler) cleanupBinlogServer(ctx context.Context, cr *apiv1.PerconaServerMySQL) error { if cr.Spec.Backup.PiTR.Enabled { return nil diff --git a/pkg/controller/ps/endpoint.go b/pkg/controller/ps/endpoint.go new file mode 100644 index 000000000..20f7e42ba --- /dev/null +++ b/pkg/controller/ps/endpoint.go @@ -0,0 +1,19 @@ +package ps + +import ( + "fmt" + "net/url" +) + +// parseEndpointURL extracts the protocol and host from an endpoint URL. +// Expected formats: "s3://s3.amazonaws.com", "https://minio-service:9000" +func parseEndpointURL(endpointURL string) (protocol, host string, err error) { + u, err := url.Parse(endpointURL) + if err != nil { + return "", "", fmt.Errorf("parse endpoint URL %q: %w", endpointURL, err) + } + if u.Host == "" { + return "", "", fmt.Errorf("endpoint URL %q must include protocol and host (e.g. s3://... or https://...)", endpointURL) + } + return u.Scheme, u.Host, nil +} diff --git a/pkg/controller/ps/endpoint_test.go b/pkg/controller/ps/endpoint_test.go new file mode 100644 index 000000000..9c553d29d --- /dev/null +++ b/pkg/controller/ps/endpoint_test.go @@ -0,0 +1,86 @@ +package ps + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseEndpointURL(t *testing.T) { + tests := []struct { + name string + input string + wantProtocol string + wantHost string + wantErr string + }{ + { + name: "https with host and port", + input: "https://minio-service:9000", + wantProtocol: "https", + wantHost: "minio-service:9000", + }, + { + name: "s3 scheme", + input: "s3://s3.amazonaws.com", + wantProtocol: "s3", + wantHost: "s3.amazonaws.com", + }, + { + name: "http scheme", + input: "http://localhost:9000", + wantProtocol: "http", + wantHost: "localhost:9000", + }, + { + name: "trailing slash is stripped", + input: "https://minio:9000/", + wantProtocol: "https", + wantHost: "minio:9000", + }, + { + name: "path after host is stripped", + input: "https://host/path/to/something", + wantProtocol: "https", + wantHost: "host", + }, + { + name: "host with port and path is stripped", + input: "https://minio:9000/bucket/prefix", + wantProtocol: "https", + wantHost: "minio:9000", + }, + { + name: "no scheme", + input: "minio-service:9000", + wantErr: "must include protocol and host", + }, + { + name: "empty string", + input: "", + wantErr: "must include protocol and host", + }, + { + name: "scheme only", + input: "https://", + wantErr: "must include protocol and host", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + protocol, host, err := parseEndpointURL(tc.input) + + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + return + } + + require.NoError(t, err) + assert.Equal(t, tc.wantProtocol, protocol) + assert.Equal(t, tc.wantHost, host) + }) + } +} From 0fddd84343ba058afc8db4cefb30cba239857f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 14 Apr 2026 15:31:43 +0300 Subject: [PATCH 085/102] apply binlogs on the fly --- cmd/pitr/main.go | 76 +++++++++++++++++-------------------------- cmd/pitr/main_test.go | 19 ++++------- 2 files changed, 36 insertions(+), 59 deletions(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 858699411..eeb3e996a 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -10,7 +10,6 @@ import ( "net/url" "os" "os/exec" - "path/filepath" "strings" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" @@ -29,9 +28,12 @@ type Database interface { type newStorageFn func(ctx context.Context, endpoint, accessKey, secretKey, bucket, prefix, region string, verifyTLS bool) (storage.Storage, error) type newDatabaseFn func(ctx context.Context, params db.DBParams) (Database, error) -// applyBinlogsFn starts a single mysql client process and for each binlog file -// runs mysqlbinlog with the given args, piping the output into mysql's stdin. -type applyBinlogsFn func(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string, mysqlPass string) error +// getObjectFn fetches a single object by key and returns a streaming reader. +type getObjectFn func(ctx context.Context, objectKey string) (io.ReadCloser, error) + +// applyBinlogsFn starts a single mysql client process and for each object key +// fetches the binlog via getObject and streams it through mysqlbinlog into mysql. +type applyBinlogsFn func(ctx context.Context, objectKeys []string, getObject getObjectFn, mysqlbinlogArgs []string, mysqlArgs []string, mysqlPass string) error func main() { ctx := context.Background() @@ -104,43 +106,13 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret return fmt.Errorf("create S3 client: %w", err) } - tmpDir, err := os.MkdirTemp("", "pitr-binlogs-") - if err != nil { - return fmt.Errorf("create temp dir: %w", err) - } - defer os.RemoveAll(tmpDir) - - var binlogPaths []string - for i, entry := range entries { - binlogName := fmt.Sprintf("binlog.%06d", i+1) - binlogPath := filepath.Join(tmpDir, binlogName) - + var objectKeys []string + for _, entry := range entries { objectKey, err := objectKeyFromURI(entry.URI, bucket) if err != nil { return fmt.Errorf("parse URI %s: %w", entry.URI, err) } - - log.Printf("downloading binlog %s to %s", objectKey, binlogPath) - - obj, err := s3Client.GetObject(ctx, objectKey) - if err != nil { - return fmt.Errorf("download binlog %s: %w", entry.URI, err) - } - - f, err := os.Create(binlogPath) - if err != nil { - obj.Close() - return fmt.Errorf("create binlog file %s: %w", binlogPath, err) - } - - _, err = io.Copy(f, obj) - obj.Close() - f.Close() - if err != nil { - return fmt.Errorf("write binlog file %s: %w", binlogPath, err) - } - - binlogPaths = append(binlogPaths, binlogPath) + objectKeys = append(objectKeys, objectKey) } // Build mysqlbinlog args. @@ -165,9 +137,9 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret "-P", "33062", } - log.Printf("applying %d binlog(s) with mysqlbinlog args: %v", len(binlogPaths), mysqlbinlogArgs) + log.Printf("applying %d binlog(s) with mysqlbinlog args: %v", len(objectKeys), mysqlbinlogArgs) - if err := apply(ctx, binlogPaths, mysqlbinlogArgs, mysqlArgs, operatorPass); err != nil { + if err := apply(ctx, objectKeys, s3Client.GetObject, mysqlbinlogArgs, mysqlArgs, operatorPass); err != nil { return fmt.Errorf("apply binlogs: %w", err) } @@ -192,9 +164,9 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret return nil } -// applyBinlogs starts a single mysql client and for each binlog file -// spawns mysqlbinlog, piping its output into mysql's stdin. -func applyBinlogs(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string, mysqlPass string) error { +// applyBinlogs starts a single mysql client and for each object key +// fetches the binlog from storage and streams it through mysqlbinlog into mysql. +func applyBinlogs(ctx context.Context, objectKeys []string, getObject getObjectFn, mysqlbinlogArgs []string, mysqlArgs []string, mysqlPass string) error { mysqlCmd := exec.CommandContext(ctx, "mysql", mysqlArgs...) mysqlCmd.Env = append(os.Environ(), fmt.Sprintf("MYSQL_PWD=%s", mysqlPass)) mysqlStdin, err := mysqlCmd.StdinPipe() @@ -209,21 +181,31 @@ func applyBinlogs(ctx context.Context, binlogPaths []string, mysqlbinlogArgs []s return fmt.Errorf("start mysql: %w", err) } - for _, binlogPath := range binlogPaths { - args := append(mysqlbinlogArgs, binlogPath) + for _, objectKey := range objectKeys { + log.Printf("streaming binlog %s", objectKey) + + obj, err := getObject(ctx, objectKey) + if err != nil { + mysqlStdin.Close() + mysqlCmd.Wait() + return fmt.Errorf("fetch binlog %s: %w", objectKey, err) + } + + args := append(mysqlbinlogArgs, "-") binlogCmd := exec.CommandContext(ctx, "mysqlbinlog", args...) + binlogCmd.Stdin = obj var binlogStderr bytes.Buffer binlogCmd.Stdout = mysqlStdin binlogCmd.Stderr = &binlogStderr - log.Printf("running: mysqlbinlog %s", binlogPath) - if err := binlogCmd.Run(); err != nil { + obj.Close() mysqlStdin.Close() mysqlCmd.Wait() - return fmt.Errorf("mysqlbinlog %s failed: %w, stderr: %s", binlogPath, err, binlogStderr.String()) + return fmt.Errorf("mysqlbinlog %s failed: %w, stderr: %s", objectKey, err, binlogStderr.String()) } + obj.Close() } mysqlStdin.Close() diff --git a/cmd/pitr/main_test.go b/cmd/pitr/main_test.go index 63f5e54ec..8bfe771c8 100644 --- a/cmd/pitr/main_test.go +++ b/cmd/pitr/main_test.go @@ -66,7 +66,7 @@ func writeBinlogsFile(t *testing.T, entries []binlogserver.BinlogEntry) string { } type applyCall struct { - binlogPaths []string + objectKeys []string mysqlbinlogArgs []string mysqlArgs []string } @@ -154,13 +154,8 @@ func TestRun(t *testing.T) { pitrType: "gtid", pitrGTID: "uuid:1", db: &fakeDB{getGTIDExecutedResult: "uuid:1-5"}, - newS3: func(fake *fakeStorage) newStorageFn { - fake.getErr = errors.New("download failed") - return func(_ context.Context, _, _, _, _, _, _ string, _ bool) (storage.Storage, error) { - return fake, nil - } - }, - expectedError: "download binlog", + applyErr: errors.New("fetch binlog binlogs/binlog.000001: download failed"), + expectedError: "apply binlogs", }, "unknown PITR type": { entries: defaultEntries, @@ -182,7 +177,7 @@ func TestRun(t *testing.T) { pitrGTID: "aaaaaaaa-0000-0000-0000-000000000001:1-10", db: &fakeDB{getGTIDExecutedResult: "aaaaaaaa-0000-0000-0000-000000000001:1-5"}, checkApply: func(t *testing.T, call applyCall) { - assert.Len(t, call.binlogPaths, 2) + assert.Len(t, call.objectKeys, 2) assert.Contains(t, call.mysqlbinlogArgs, "--disable-log-bin") assert.Contains(t, call.mysqlbinlogArgs, "--exclude-gtids=aaaaaaaa-0000-0000-0000-000000000001:1-5") assert.Contains(t, call.mysqlbinlogArgs, "--include-gtids=aaaaaaaa-0000-0000-0000-000000000001:1-10") @@ -195,7 +190,7 @@ func TestRun(t *testing.T) { pitrDate: "2024-01-15 12:00:00", db: &fakeDB{getGTIDExecutedResult: "bbbbbbbb-0000-0000-0000-000000000002:1-5"}, checkApply: func(t *testing.T, call applyCall) { - assert.Len(t, call.binlogPaths, 2) + assert.Len(t, call.objectKeys, 2) assert.Contains(t, call.mysqlbinlogArgs, "--disable-log-bin") assert.Contains(t, call.mysqlbinlogArgs, "--exclude-gtids=bbbbbbbb-0000-0000-0000-000000000002:1-5") assert.Contains(t, call.mysqlbinlogArgs, "--stop-datetime=2024-01-15 12:00:00") @@ -269,9 +264,9 @@ func TestRun(t *testing.T) { } var captured applyCall - apply := func(_ context.Context, binlogPaths []string, mysqlbinlogArgs []string, mysqlArgs []string, _ string) error { + apply := func(_ context.Context, objectKeys []string, _ getObjectFn, mysqlbinlogArgs []string, mysqlArgs []string, _ string) error { captured = applyCall{ - binlogPaths: binlogPaths, + objectKeys: objectKeys, mysqlbinlogArgs: mysqlbinlogArgs, mysqlArgs: mysqlArgs, } From aaf2e868914732e459d6b455e8e5c27ac2e93874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 14 Apr 2026 15:39:22 +0300 Subject: [PATCH 086/102] log Close and Wait errors --- cmd/pitr/main.go | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index eeb3e996a..f62342b18 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -88,11 +88,15 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret gtidExecuted, err := database.GetGTIDExecuted(ctx) if err != nil { - database.Close() + if closeErr := database.Close(); closeErr != nil { + log.Printf("close database: %v", closeErr) + } return fmt.Errorf("get GTID_EXECUTED: %w", err) } log.Printf("GTID_EXECUTED from backup: %s", gtidExecuted) - database.Close() + if err := database.Close(); err != nil { + log.Printf("close database: %v", err) + } endpoint := os.Getenv("AWS_ENDPOINT") accessKey := os.Getenv("AWS_ACCESS_KEY_ID") @@ -186,8 +190,12 @@ func applyBinlogs(ctx context.Context, objectKeys []string, getObject getObjectF obj, err := getObject(ctx, objectKey) if err != nil { - mysqlStdin.Close() - mysqlCmd.Wait() + if closeErr := mysqlStdin.Close(); closeErr != nil { + log.Printf("close mysql stdin: %v", closeErr) + } + if waitErr := mysqlCmd.Wait(); waitErr != nil { + log.Printf("wait for mysql: %v", waitErr) + } return fmt.Errorf("fetch binlog %s: %w", objectKey, err) } @@ -200,15 +208,25 @@ func applyBinlogs(ctx context.Context, objectKeys []string, getObject getObjectF binlogCmd.Stderr = &binlogStderr if err := binlogCmd.Run(); err != nil { - obj.Close() - mysqlStdin.Close() - mysqlCmd.Wait() + if closeErr := obj.Close(); closeErr != nil { + log.Printf("close object %s: %v", objectKey, closeErr) + } + if closeErr := mysqlStdin.Close(); closeErr != nil { + log.Printf("close mysql stdin: %v", closeErr) + } + if waitErr := mysqlCmd.Wait(); waitErr != nil { + log.Printf("wait for mysql: %v", waitErr) + } return fmt.Errorf("mysqlbinlog %s failed: %w, stderr: %s", objectKey, err, binlogStderr.String()) } - obj.Close() + if err := obj.Close(); err != nil { + log.Printf("close object %s: %v", objectKey, err) + } } - mysqlStdin.Close() + if err := mysqlStdin.Close(); err != nil { + log.Printf("close mysql stdin: %v", err) + } if err := mysqlCmd.Wait(); err != nil { return fmt.Errorf("mysql failed: %w, stderr: %s", err, mysqlStderr.String()) From 4d423334c15c910ffe138939219f2d026a8cd094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 14 Apr 2026 16:54:59 +0300 Subject: [PATCH 087/102] add async test --- e2e-tests/run-distro.csv | 1 + e2e-tests/run-pr.csv | 1 + e2e-tests/run-release.csv | 1 + .../tests/async-pitr-minio/00-assert.yaml | 9 +++ .../async-pitr-minio/00-minio-secret.yaml | 7 ++ .../tests/async-pitr-minio/01-assert.yaml | 26 +++++++ .../async-pitr-minio/01-deploy-operator.yaml | 15 ++++ .../tests/async-pitr-minio/02-assert.yaml | 70 +++++++++++++++++++ .../async-pitr-minio/02-create-cluster.yaml | 38 ++++++++++ .../tests/async-pitr-minio/03-write-data.yaml | 16 +++++ .../tests/async-pitr-minio/04-assert.yaml | 12 ++++ .../async-pitr-minio/04-create-backup.yaml | 9 +++ .../async-pitr-minio/05-write-more-data.yaml | 12 ++++ .../tests/async-pitr-minio/06-assert.yaml | 35 ++++++++++ .../06-create-pitr-restore.yaml | 39 +++++++++++ .../tests/async-pitr-minio/07-assert.yaml | 24 +++++++ .../tests/async-pitr-minio/07-read-data.yaml | 16 +++++ .../tests/async-pitr-minio/08-assert.yaml | 31 ++++++++ .../08-create-date-restore.yaml | 23 ++++++ .../tests/async-pitr-minio/09-assert.yaml | 24 +++++++ .../tests/async-pitr-minio/09-read-data.yaml | 16 +++++ .../tests/async-pitr-minio/10-failover.yaml | 37 ++++++++++ .../tests/async-pitr-minio/11-assert.yaml | 31 ++++++++ .../async-pitr-minio/11-write-and-backup.yaml | 24 +++++++ .../tests/async-pitr-minio/12-assert.yaml | 35 ++++++++++ .../12-write-and-restore.yaml | 30 ++++++++ .../tests/async-pitr-minio/13-assert.yaml | 24 +++++++ .../tests/async-pitr-minio/13-read-data.yaml | 16 +++++ .../async-pitr-minio/98-drop-finalizer.yaml | 5 ++ .../99-remove-cluster-gracefully.yaml | 16 +++++ 30 files changed, 643 insertions(+) create mode 100644 e2e-tests/tests/async-pitr-minio/00-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/00-minio-secret.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/01-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/01-deploy-operator.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/02-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/02-create-cluster.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/03-write-data.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/04-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/04-create-backup.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/05-write-more-data.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/06-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/07-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/07-read-data.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/08-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/08-create-date-restore.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/09-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/09-read-data.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/10-failover.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/11-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/12-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/12-write-and-restore.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/13-assert.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/13-read-data.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/98-drop-finalizer.yaml create mode 100644 e2e-tests/tests/async-pitr-minio/99-remove-cluster-gracefully.yaml diff --git a/e2e-tests/run-distro.csv b/e2e-tests/run-distro.csv index ea39a814e..8e284f453 100644 --- a/e2e-tests/run-distro.csv +++ b/e2e-tests/run-distro.csv @@ -4,6 +4,7 @@ config config-router demand-backup-minio gr-pitr-minio +async-pitr-minio demand-backup-cloud demand-backup-retry gr-demand-backup-minio diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 564d92a54..5256c4480 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -9,6 +9,7 @@ config-router,8.4 demand-backup-minio,8.0 demand-backup-minio,8.4 gr-pitr-minio,8.4 +async-pitr-minio,8.4 demand-backup-cloud,8.4 demand-backup-retry,8.4 demand-backup-incremental,8.0 diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index 3d5300133..563caff2b 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -6,6 +6,7 @@ config config-router demand-backup-minio gr-pitr-minio +async-pitr-minio demand-backup-cloud demand-backup-retry demand-backup-incremental diff --git a/e2e-tests/tests/async-pitr-minio/00-assert.yaml b/e2e-tests/tests/async-pitr-minio/00-assert.yaml new file mode 100644 index 000000000..fecdb222e --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/00-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 150 +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +type: Opaque diff --git a/e2e-tests/tests/async-pitr-minio/00-minio-secret.yaml b/e2e-tests/tests/async-pitr-minio/00-minio-secret.yaml new file mode 100644 index 000000000..3c797f054 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/00-minio-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +stringData: + AWS_ACCESS_KEY_ID: some-access$\n"-key + AWS_SECRET_ACCESS_KEY: some-$\n"secret-key diff --git a/e2e-tests/tests/async-pitr-minio/01-assert.yaml b/e2e-tests/tests/async-pitr-minio/01-assert.yaml new file mode 100644 index 000000000..5f346cb51 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/01-assert.yaml @@ -0,0 +1,26 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 150 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: perconaservermysqls.ps.percona.com +spec: + group: ps.percona.com + names: + kind: PerconaServerMySQL + listKind: PerconaServerMySQLList + plural: perconaservermysqls + shortNames: + - ps + singular: perconaservermysql + scope: Namespaced +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/async-pitr-minio/01-deploy-operator.yaml b/e2e-tests/tests/async-pitr-minio/01-deploy-operator.yaml new file mode 100644 index 000000000..6ab1b37f9 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/01-deploy-operator.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + apply_s3_storage_secrets + deploy_operator + deploy_client + deploy_minio + timeout: 300 diff --git a/e2e-tests/tests/async-pitr-minio/02-assert.yaml b/e2e-tests/tests/async-pitr-minio/02-assert.yaml new file mode 100644 index 000000000..a5da7bf0e --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/02-assert.yaml @@ -0,0 +1,70 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-pitr-minio-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-pitr-minio-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + updatedReplicas: 3 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-pitr-minio-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: async-pitr-minio-binlog-server +status: + observedGeneration: 1 + replicas: 1 + readyReplicas: 1 + currentReplicas: 1 + updatedReplicas: 1 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: async-pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + mysql: + ready: 3 + size: 3 + state: ready + haproxy: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/async-pitr-minio/02-create-cluster.yaml b/e2e-tests/tests/async-pitr-minio/02-create-cluster.yaml new file mode 100644 index 000000000..ffd6f4b93 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/02-create-cluster.yaml @@ -0,0 +1,38 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval '.spec.mysql.clusterType="async"' - \ + | yq eval ".spec.mysql.size=3" - \ + | yq eval ".spec.orchestrator.enabled=true" - \ + | yq eval ".spec.proxy.haproxy.enabled=true" - \ + | yq eval ".spec.proxy.haproxy.size=3" - \ + | yq eval ".spec.proxy.router.enabled=false" - \ + | yq eval ".spec.backup.backoffLimit=3" - \ + | yq eval '.spec.backup.storages.minio.type="s3"' - \ + | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.minio.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.storages.minio.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.storages.minio.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.storages.minio.containerOptions.env[0].name="VERIFY_TLS"' - \ + | yq eval '.spec.backup.storages.minio.containerOptions.env[0].value="false"' - \ + | yq eval '.spec.backup.pitr.enabled=true' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.prefix="binlogs"' - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.pitr.binlogServer.storage.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.pitr.binlogServer.storage.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.pitr.binlogServer.size=1' - \ + | yq eval '.spec.backup.pitr.binlogServer.serverId=100' - \ + | yq eval '.spec.backup.pitr.binlogServer.connectTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.readTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.writeTimeout=10' - \ + | yq eval '.spec.backup.pitr.binlogServer.idleTime=3' - \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/async-pitr-minio/03-write-data.yaml b/e2e-tests/tests/async-pitr-minio/03-write-data.yaml new file mode 100644 index 000000000..d490db677 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/03-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ + "-h $(get_haproxy_svc $(get_cluster_name))" + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100500)" \ + "-h $(get_haproxy_svc $(get_cluster_name))" diff --git a/e2e-tests/tests/async-pitr-minio/04-assert.yaml b/e2e-tests/tests/async-pitr-minio/04-assert.yaml new file mode 100644 index 000000000..4bcdcacd7 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/04-assert.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1 +metadata: + name: async-pitr-minio-backup + finalizers: + - percona.com/delete-backup +status: + state: Succeeded diff --git a/e2e-tests/tests/async-pitr-minio/04-create-backup.yaml b/e2e-tests/tests/async-pitr-minio/04-create-backup.yaml new file mode 100644 index 000000000..38d73a87d --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/04-create-backup.yaml @@ -0,0 +1,9 @@ +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQLBackup +metadata: + name: async-pitr-minio-backup + finalizers: + - percona.com/delete-backup +spec: + clusterName: async-pitr-minio + storageName: minio diff --git a/e2e-tests/tests/async-pitr-minio/05-write-more-data.yaml b/e2e-tests/tests/async-pitr-minio/05-write-more-data.yaml new file mode 100644 index 000000000..ec30f75d5 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/05-write-more-data.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100501)" \ + "-h $(get_haproxy_svc $(get_cluster_name))" diff --git a/e2e-tests/tests/async-pitr-minio/06-assert.yaml b/e2e-tests/tests/async-pitr-minio/06-assert.yaml new file mode 100644 index 000000000..90ee0b7eb --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/06-assert.yaml @@ -0,0 +1,35 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +collectors: + - type: pod + selector: "app.kubernetes.io/component=pitr" + tail: 1000 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: async-pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + mysql: + ready: 3 + size: 3 + state: ready + haproxy: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +kind: PerconaServerMySQLRestore +apiVersion: ps.percona.com/v1 +metadata: + name: async-pitr-minio-restore +status: + state: Succeeded diff --git a/e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml new file mode 100644 index 000000000..3e331529d --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml @@ -0,0 +1,39 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 90 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + GTID_BEFORE=$(run_mysql "SELECT @@GLOBAL.gtid_executed" "-h $(get_haproxy_svc ${cluster_name})" | tr -d '\n') + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100502)" \ + "-h $(get_haproxy_svc ${cluster_name})" + + PITR_GTID=$(run_mysql "SELECT GTID_SUBTRACT(@@GLOBAL.gtid_executed, '${GTID_BEFORE}')" "-h $(get_haproxy_svc ${cluster_name})" | tr -d '\n') + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100503)" \ + "-h $(get_haproxy_svc ${cluster_name})" + + sleep 10 + PITR_DATE=$(date -u '+%Y-%m-%d %H:%M:%S') + kubectl create configmap -n "${NAMESPACE}" pitr-date --from-literal=date="${PITR_DATE}" + + sleep 60 + + echo '{}' \ + | yq eval '.apiVersion = "ps.percona.com/v1"' - \ + | yq eval '.kind = "PerconaServerMySQLRestore"' - \ + | yq eval '.metadata.name = "async-pitr-minio-restore"' - \ + | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ + | yq eval '.spec.backupName = "async-pitr-minio-backup"' - \ + | yq eval '.spec.pitr.type = "gtid"' - \ + | yq eval ".spec.pitr.gtid = \"${PITR_GTID}\"" - \ + | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/async-pitr-minio/07-assert.yaml b/e2e-tests/tests/async-pitr-minio/07-assert.yaml new file mode 100644 index 000000000..c2d3d8132 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/07-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-data-0 +data: + max_id: "100502" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-data-1 +data: + max_id: "100502" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-data-2 +data: + max_id: "100502" diff --git a/e2e-tests/tests/async-pitr-minio/07-read-data.yaml b/e2e-tests/tests/async-pitr-minio/07-read-data.yaml new file mode 100644 index 000000000..68ba007c9 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/07-read-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + for i in 0 1 2; do + max_id=$(run_mysql "SELECT MAX(id) FROM myDB.myTable" "-h ${cluster_name}-mysql-${i}.${cluster_name}-mysql") + kubectl create configmap -n "${NAMESPACE}" 07-read-data-${i} --from-literal=max_id="${max_id}" + done diff --git a/e2e-tests/tests/async-pitr-minio/08-assert.yaml b/e2e-tests/tests/async-pitr-minio/08-assert.yaml new file mode 100644 index 000000000..25082063c --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/08-assert.yaml @@ -0,0 +1,31 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: async-pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + mysql: + ready: 3 + size: 3 + state: ready + haproxy: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +kind: PerconaServerMySQLRestore +apiVersion: ps.percona.com/v1 +metadata: + name: async-pitr-minio-restore-date +status: + state: Succeeded diff --git a/e2e-tests/tests/async-pitr-minio/08-create-date-restore.yaml b/e2e-tests/tests/async-pitr-minio/08-create-date-restore.yaml new file mode 100644 index 000000000..485147a9a --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/08-create-date-restore.yaml @@ -0,0 +1,23 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 90 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + PITR_DATE=$(kubectl get configmap -n "${NAMESPACE}" pitr-date -o jsonpath='{.data.date}') + + echo '{}' \ + | yq eval '.apiVersion = "ps.percona.com/v1"' - \ + | yq eval '.kind = "PerconaServerMySQLRestore"' - \ + | yq eval '.metadata.name = "async-pitr-minio-restore-date"' - \ + | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ + | yq eval '.spec.backupName = "async-pitr-minio-backup"' - \ + | yq eval '.spec.pitr.type = "date"' - \ + | yq eval ".spec.pitr.date = \"${PITR_DATE}\"" - \ + | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/async-pitr-minio/09-assert.yaml b/e2e-tests/tests/async-pitr-minio/09-assert.yaml new file mode 100644 index 000000000..8f6ce88d6 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/09-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 09-read-data-0 +data: + max_id: "100503" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 09-read-data-1 +data: + max_id: "100503" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 09-read-data-2 +data: + max_id: "100503" diff --git a/e2e-tests/tests/async-pitr-minio/09-read-data.yaml b/e2e-tests/tests/async-pitr-minio/09-read-data.yaml new file mode 100644 index 000000000..8d79b4d3f --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/09-read-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + for i in 0 1 2; do + max_id=$(run_mysql "SELECT MAX(id) FROM myDB.myTable" "-h ${cluster_name}-mysql-${i}.${cluster_name}-mysql") + kubectl create configmap -n "${NAMESPACE}" 09-read-data-${i} --from-literal=max_id="${max_id}" + done diff --git a/e2e-tests/tests/async-pitr-minio/10-failover.yaml b/e2e-tests/tests/async-pitr-minio/10-failover.yaml new file mode 100644 index 000000000..276518152 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/10-failover.yaml @@ -0,0 +1,37 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 600 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + primary_before=$(get_primary_from_label) + + kubectl -n "${NAMESPACE}" delete pod "${primary_before}" + + wait_cluster_consistency_async "${cluster_name}" 3 3 + + primary_after=$(get_primary_from_label) + + if [[ "${primary_before}" == "${primary_after}" ]]; then + echo "Primary pod did not change after failover: was ${primary_before}, still ${primary_after}" + exit 1 + fi + echo "Primary changed from ${primary_before} to ${primary_after}" + + retry=0 + until [[ "$(kubectl -n "${NAMESPACE}" get endpoints "${cluster_name}-mysql-primary" \ + -o jsonpath='{.subsets[0].addresses[0].targetRef.name}' 2>/dev/null)" == "${primary_after}" ]]; do + sleep 5 + retry=$((retry + 1)) + if [ $retry -ge 24 ]; then + echo "Primary service endpoint did not update to ${primary_after} after 2 minutes" + kubectl -n "${NAMESPACE}" get endpoints "${cluster_name}-mysql-primary" -o yaml + exit 1 + fi + done + echo "Primary service endpoint correctly points to new primary: ${primary_after}" diff --git a/e2e-tests/tests/async-pitr-minio/11-assert.yaml b/e2e-tests/tests/async-pitr-minio/11-assert.yaml new file mode 100644 index 000000000..3d0975a28 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/11-assert.yaml @@ -0,0 +1,31 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: async-pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + mysql: + ready: 3 + size: 3 + state: ready + haproxy: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1 +metadata: + name: async-pitr-minio-backup-2 +status: + state: Succeeded diff --git a/e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml b/e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml new file mode 100644 index 000000000..69f9161be --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (200500)" \ + "-h $(get_haproxy_svc ${cluster_name})" + + echo '{}' \ + | yq eval '.apiVersion = "ps.percona.com/v1"' - \ + | yq eval '.kind = "PerconaServerMySQLBackup"' - \ + | yq eval '.metadata.name = "async-pitr-minio-backup-2"' - \ + | yq eval '.metadata.finalizers[0] = "percona.com/delete-backup"' - \ + | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ + | yq eval '.spec.storageName = "minio"' - \ + | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/async-pitr-minio/12-assert.yaml b/e2e-tests/tests/async-pitr-minio/12-assert.yaml new file mode 100644 index 000000000..abb266d09 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/12-assert.yaml @@ -0,0 +1,35 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +collectors: + - type: pod + selector: "app.kubernetes.io/component=pitr" + tail: 1000 +--- +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: async-pitr-minio + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + mysql: + ready: 3 + size: 3 + state: ready + haproxy: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready +--- +kind: PerconaServerMySQLRestore +apiVersion: ps.percona.com/v1 +metadata: + name: async-pitr-minio-restore-post-failover +status: + state: Succeeded diff --git a/e2e-tests/tests/async-pitr-minio/12-write-and-restore.yaml b/e2e-tests/tests/async-pitr-minio/12-write-and-restore.yaml new file mode 100644 index 000000000..93a6157c1 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/12-write-and-restore.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 600 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (200501)" \ + "-h $(get_haproxy_svc ${cluster_name})" + + sleep 120 + + PITR_DATE_POST=$(date -u '+%Y-%m-%d %H:%M:%S') + kubectl create configmap -n "${NAMESPACE}" pitr-date-post --from-literal=date="${PITR_DATE_POST}" + + echo '{}' \ + | yq eval '.apiVersion = "ps.percona.com/v1"' - \ + | yq eval '.kind = "PerconaServerMySQLRestore"' - \ + | yq eval '.metadata.name = "async-pitr-minio-restore-post-failover"' - \ + | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ + | yq eval '.spec.backupName = "async-pitr-minio-backup-2"' - \ + | yq eval '.spec.pitr.type = "date"' - \ + | yq eval ".spec.pitr.date = \"${PITR_DATE_POST}\"" - \ + | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/async-pitr-minio/13-assert.yaml b/e2e-tests/tests/async-pitr-minio/13-assert.yaml new file mode 100644 index 000000000..a771c8bac --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/13-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-data-0 +data: + max_id: "200501" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-data-1 +data: + max_id: "200501" +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 13-read-data-2 +data: + max_id: "200501" diff --git a/e2e-tests/tests/async-pitr-minio/13-read-data.yaml b/e2e-tests/tests/async-pitr-minio/13-read-data.yaml new file mode 100644 index 000000000..1034b3210 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/13-read-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + cluster_name=$(get_cluster_name) + + for i in 0 1 2; do + max_id=$(run_mysql "SELECT MAX(id) FROM myDB.myTable" "-h ${cluster_name}-mysql-${i}.${cluster_name}-mysql") + kubectl create configmap -n "${NAMESPACE}" 13-read-data-${i} --from-literal=max_id="${max_id}" + done diff --git a/e2e-tests/tests/async-pitr-minio/98-drop-finalizer.yaml b/e2e-tests/tests/async-pitr-minio/98-drop-finalizer.yaml new file mode 100644 index 000000000..ce815f41f --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/98-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1 +kind: PerconaServerMySQL +metadata: + name: async-pitr-minio + finalizers: [] diff --git a/e2e-tests/tests/async-pitr-minio/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/async-pitr-minio/99-remove-cluster-gracefully.yaml new file mode 100644 index 000000000..b54d33c84 --- /dev/null +++ b/e2e-tests/tests/async-pitr-minio/99-remove-cluster-gracefully.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: ps.percona.com/v1 + kind: PerconaServerMySQL + metadata: + name: async-pitr-minio +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + destroy_operator + timeout: 180 From 86ee68b27594215b5707d0dd447b6f552d31bede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Wed, 15 Apr 2026 14:41:41 +0300 Subject: [PATCH 088/102] use mysql log format for pitr --- build/run-pitr-restore.sh | 23 +++++++++++++++-------- cmd/pitr/main.go | 13 +++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/build/run-pitr-restore.sh b/build/run-pitr-restore.sh index 96b1dcd16..f4a5effed 100644 --- a/build/run-pitr-restore.sh +++ b/build/run-pitr-restore.sh @@ -2,30 +2,37 @@ set -e -# TODO: Add support for data at rest encryption +function log() { + local ts=$(date +%Y-%m-%dT%H:%M:%S.%N%z --utc | sed 's/+0000/Z/g') + echo "${ts} 0 [Info] [K8SPS-642] [Job] $*" >&2 +} -echo "Starting mysqld" +log "Starting mysqld" +# TODO: Add support for data at rest encryption mysqld \ --admin-address=127.0.0.1 \ --user=mysql \ --gtid-mode=ON \ - --enforce-gtid-consistency=ON >/tmp/mysqld.log 2>&1 & + --enforce-gtid-consistency=ON & -echo "Waiting for mysqld to be ready" +log "waiting for mysqld to be ready" until mysqladmin -u operator -p"$(/dev/null; do sleep 1; done +log "mysqld is ready" if [[ -n ${SLEEP_FOREVER} ]]; then - echo "Sleeping forever..." - touch /var/lib/mysql/sleep-forever - while [[ -f /var/lib/mysql/sleep-forever ]]; do + SLEEP_FOREVER_FILE=/var/lib/mysql/sleep-forever + log "sleeping forever... remove ${SLEEP_FOREVER_FILE} to terminate." + touch ${SLEEP_FOREVER_FILE} + while [[ -f ${SLEEP_FOREVER_FILE} ]]; do sleep 10 done exit 0 fi +log "starting recovery" /opt/percona/pitr -echo "Stopping mysqld" +log "stopping mysqld" mysqladmin -u operator -p"$(/dev/null diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index f62342b18..5e9220402 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "strings" + "time" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" "github.com/percona/percona-server-mysql-operator/cmd/bootstrap/utils" @@ -35,9 +36,21 @@ type getObjectFn func(ctx context.Context, objectKey string) (io.ReadCloser, err // fetches the binlog via getObject and streams it through mysqlbinlog into mysql. type applyBinlogsFn func(ctx context.Context, objectKeys []string, getObject getObjectFn, mysqlbinlogArgs []string, mysqlArgs []string, mysqlPass string) error +type logWriter struct{} + +func (lw *logWriter) Write(bs []byte) (int, error) { + return fmt.Print(time.Now().UTC().Format(time.RFC3339Nano), " 0 [Info] [K8SPS-642] [Recovery] ", string(bs)) +} + func main() { ctx := context.Background() + // we use a custom writer to match mysqld log format. + // mysqld and pitr logs are printed to together to stdout/stderr + // and it should be possible to parse them together + log.SetFlags(0) + log.SetOutput(new(logWriter)) + newDB := func(ctx context.Context, params db.DBParams) (Database, error) { return db.NewDatabase(ctx, params) } From 180f8da1d096f58a956b916a40eb791429f9f081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Wed, 15 Apr 2026 16:53:17 +0300 Subject: [PATCH 089/102] use --force to apply binlogs --- cmd/pitr/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 5e9220402..686632d9b 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -149,6 +149,7 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret // Build mysql client args. mysqlArgs := []string{ + "--force", "-u", string(apiv1.UserOperator), "-h", "127.0.0.1", "-P", "33062", From 1631578131ab1684fc314a3f0f151f88ab1141a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Wed, 15 Apr 2026 17:29:40 +0300 Subject: [PATCH 090/102] fix async test --- e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml b/e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml index 3e331529d..654e7503c 100644 --- a/e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml +++ b/e2e-tests/tests/async-pitr-minio/06-create-pitr-restore.yaml @@ -1,7 +1,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - timeout: 90 + - timeout: 120 script: |- set -o errexit set -o xtrace From 8b1acff33747e8fc454ed92db4475cd1787874b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:20:31 +0300 Subject: [PATCH 091/102] fix pausing cluster with one pod --- pkg/controller/ps/controller.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index cb08f9b7e..f726668ec 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -827,7 +827,7 @@ func (r *PerconaServerMySQLReconciler) reconcileHAProxy(ctx context.Context, cr return errors.Wrapf(err, "check if pod %s ready", nn.String()) } - if !firstMySQLPodReady { + if !firstMySQLPodReady && !cr.Spec.Pause { log.V(1).Info("Waiting for pod to be ready", "pod", nn.Name) return nil } @@ -986,7 +986,7 @@ func (r *PerconaServerMySQLReconciler) reconcileGroupReplication(ctx context.Con } func (r *PerconaServerMySQLReconciler) reconcileBootstrapStatus(ctx context.Context, cr *apiv1.PerconaServerMySQL) error { - log := logf.FromContext(ctx) + log := logf.FromContext(ctx).WithName("Bootstrap") if cr.Status.MySQL.Ready == 0 || cr.Status.MySQL.Ready != cr.Spec.MySQL.Size { log.V(1).Info("Waiting for all MySQL pods to be ready", "ready", cr.Status.MySQL.Ready, "expected", cr.Spec.MySQL.Size) @@ -1299,7 +1299,7 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context return nil } - logger := logf.FromContext(ctx) + logger := logf.FromContext(ctx).WithName("BinlogServer") s3 := cr.Spec.Backup.PiTR.BinlogServer.Storage.S3 @@ -1359,7 +1359,7 @@ func (r *PerconaServerMySQLReconciler) reconcileBinlogServer(ctx context.Context ConnectTimeout: cr.Spec.Backup.PiTR.BinlogServer.ConnectTimeout, WriteTimeout: cr.Spec.Backup.PiTR.BinlogServer.WriteTimeout, ReadTimeout: cr.Spec.Backup.PiTR.BinlogServer.ReadTimeout, - SSL: binlogServerSSLConfig(cr.Spec.Backup.PiTR.BinlogServer.SSLMode), + SSL: binlogServerSSLConfig(cr.Spec.Backup.PiTR.BinlogServer.SSLMode), }, Replication: binlogserver.Replication{ Mode: binlogserver.ReplicationModeGTID, From 62adfcd6285043ce3e4fad10ff2d47f9bb129de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:23:41 +0300 Subject: [PATCH 092/102] remove unused code --- cmd/internal/db/db.go | 50 ------------------------------------------- cmd/pitr/main.go | 4 ---- 2 files changed, 54 deletions(-) diff --git a/cmd/internal/db/db.go b/cmd/internal/db/db.go index d985ef02a..68ea9dd1d 100644 --- a/cmd/internal/db/db.go +++ b/cmd/internal/db/db.go @@ -412,53 +412,3 @@ func (d *DB) GetGTIDExecuted(ctx context.Context) (string, error) { err := d.db.QueryRowContext(ctx, "SELECT @@GTID_EXECUTED").Scan(>id) return gtid, errors.Wrap(err, "get GTID_EXECUTED") } - -func (d *DB) SetGTIDNextAutomatic(ctx context.Context) error { - _, err := d.db.ExecContext(ctx, "SET GTID_NEXT='AUTOMATIC'") - return errors.Wrap(err, "set GTID_NEXT to AUTOMATIC") -} - -func (d *DB) WaitReplicaSQLThreadStop(ctx context.Context, pollInterval time.Duration) error { - for { - var serviceState string - err := d.db.QueryRowContext(ctx, - "SELECT SERVICE_STATE FROM replication_applier_status WHERE CHANNEL_NAME=''").Scan(&serviceState) - if err != nil { - return errors.Wrap(err, "query replication applier status") - } - - if serviceState == "OFF" { - break - } - - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(pollInterval): - } - } - - rows, err := d.db.QueryContext(ctx, - "SELECT LAST_ERROR_NUMBER, LAST_ERROR_MESSAGE FROM replication_applier_status_by_worker WHERE CHANNEL_NAME=''") - if err != nil { - return errors.Wrap(err, "query replication applier worker status") - } - defer func() { - if err := rows.Close(); err != nil { - logf.FromContext(ctx).Error(err, "close rows") - } - }() - - for rows.Next() { - var errNum int - var errMsg string - if err := rows.Scan(&errNum, &errMsg); err != nil { - return errors.Wrap(err, "scan worker status") - } - if errNum != 0 { - return errors.Errorf("replication worker error %d: %s", errNum, errMsg) - } - } - - return rows.Err() -} diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 686632d9b..264a505ec 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -84,7 +84,6 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret pitrDate := os.Getenv("PITR_DATE") pitrGTID := os.Getenv("PITR_GTID") - // Connect to MySQL to get the backup's GTID_EXECUTED. operatorPass, err := getSecret(apiv1.UserOperator) if err != nil { return fmt.Errorf("get operator password: %w", err) @@ -132,7 +131,6 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret objectKeys = append(objectKeys, objectKey) } - // Build mysqlbinlog args. mysqlbinlogArgs := []string{"--disable-log-bin"} if gtidExecuted != "" { mysqlbinlogArgs = append(mysqlbinlogArgs, fmt.Sprintf("--exclude-gtids=%s", gtidExecuted)) @@ -147,7 +145,6 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret return fmt.Errorf("unknown PITR_TYPE: %s", pitrType) } - // Build mysql client args. mysqlArgs := []string{ "--force", "-u", string(apiv1.UserOperator), @@ -161,7 +158,6 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret return fmt.Errorf("apply binlogs: %w", err) } - // Reconnect to log the final GTID state. database, err = newDB(ctx, db.DBParams{ User: apiv1.UserOperator, Pass: operatorPass, From 3c79b9af147cfa5cd64ce8424b1669c7d2ee62f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:25:07 +0300 Subject: [PATCH 093/102] fix gr-tls-cert-manager --- e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml | 6 +++--- e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml b/e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml index c9ec13821..43935ad40 100644 --- a/e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml +++ b/e2e-tests/tests/gr-tls-cert-manager/03-check-cert.yaml @@ -12,9 +12,9 @@ commands: "*.gr-tls-cert-manager-mysql", "*.gr-tls-cert-manager-mysql.'"${NAMESPACE}"'", "*.gr-tls-cert-manager-mysql.'"${NAMESPACE}"'.svc", - "tls-cert-manager-mysql-primary", - "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'", - "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'.svc", + "gr-tls-cert-manager-mysql-primary", + "gr-tls-cert-manager-mysql-primary.'"${NAMESPACE}"'", + "gr-tls-cert-manager-mysql-primary.'"${NAMESPACE}"'.svc", "*.gr-tls-cert-manager-orchestrator", "*.gr-tls-cert-manager-orchestrator.'"${NAMESPACE}"'", "*.gr-tls-cert-manager-orchestrator.'"${NAMESPACE}"'.svc", diff --git a/e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml b/e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml index 84e4ea121..1542b9f86 100644 --- a/e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml +++ b/e2e-tests/tests/gr-tls-cert-manager/05-check-cert.yaml @@ -12,9 +12,9 @@ commands: "*.gr-tls-cert-manager-mysql", "*.gr-tls-cert-manager-mysql.'"${NAMESPACE}"'", "*.gr-tls-cert-manager-mysql.'"${NAMESPACE}"'.svc", - "tls-cert-manager-mysql-primary", - "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'", - "tls-cert-manager-mysql-primary.'"${NAMESPACE}"'.svc", + "gr-tls-cert-manager-mysql-primary", + "gr-tls-cert-manager-mysql-primary.'"${NAMESPACE}"'", + "gr-tls-cert-manager-mysql-primary.'"${NAMESPACE}"'.svc", "*.gr-tls-cert-manager-orchestrator", "*.gr-tls-cert-manager-orchestrator.'"${NAMESPACE}"'", "*.gr-tls-cert-manager-orchestrator.'"${NAMESPACE}"'.svc", From 6f4795a672ffbf28bb90de376a020c4d691ba7f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:26:35 +0300 Subject: [PATCH 094/102] remove more unused code --- cmd/internal/db/db.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cmd/internal/db/db.go b/cmd/internal/db/db.go index 68ea9dd1d..bc5994681 100644 --- a/cmd/internal/db/db.go +++ b/cmd/internal/db/db.go @@ -395,18 +395,6 @@ func (d *DB) EnableSuperReadonly(ctx context.Context) error { return errors.Wrap(err, "set global super_read_only param to 1") } -func (d *DB) ResetBinaryLogAndGTIDs(ctx context.Context) error { - _, err := d.db.ExecContext(ctx, "RESET BINARY LOGS AND GTIDS") - return errors.Wrap(err, "reset binary logs and gtids") -} - -func (d *DB) StartReplicaUntilPosition(ctx context.Context, relayLogFile string, relayLogPos int) error { - _, err := d.db.ExecContext(ctx, fmt.Sprintf( - "START REPLICA SQL_THREAD UNTIL RELAY_LOG_FILE='%s', RELAY_LOG_POS=%d", - relayLogFile, relayLogPos)) - return errors.Wrap(err, "start replica until position") -} - func (d *DB) GetGTIDExecuted(ctx context.Context) (string, error) { var gtid string err := d.db.QueryRowContext(ctx, "SELECT @@GTID_EXECUTED").Scan(>id) From c77b10f425ef9e8719e49a0f64f3042ec07f4c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:28:24 +0300 Subject: [PATCH 095/102] fix linter --- cmd/pitr/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index 264a505ec..fbca1eea1 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -166,7 +166,11 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret if err != nil { return fmt.Errorf("reconnect to MySQL: %w", err) } - defer database.Close() + defer func() { + if err := database.Close(); err != nil { + log.Printf("close db connection: %v", err) + } + }() gtidExecuted, err = database.GetGTIDExecuted(ctx) if err != nil { From d86fe9fada688c41697fa601538649a50d92a426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:31:00 +0300 Subject: [PATCH 096/102] fix pitr tests --- e2e-tests/tests/async-pitr-minio/04-assert.yaml | 2 -- e2e-tests/tests/async-pitr-minio/04-create-backup.yaml | 2 -- e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml | 1 - e2e-tests/tests/gr-pitr-minio/04-assert.yaml | 2 -- e2e-tests/tests/gr-pitr-minio/04-create-backup.yaml | 2 -- e2e-tests/tests/gr-pitr-minio/11-write-and-backup.yaml | 3 +-- 6 files changed, 1 insertion(+), 11 deletions(-) diff --git a/e2e-tests/tests/async-pitr-minio/04-assert.yaml b/e2e-tests/tests/async-pitr-minio/04-assert.yaml index 4bcdcacd7..f6f17bee2 100644 --- a/e2e-tests/tests/async-pitr-minio/04-assert.yaml +++ b/e2e-tests/tests/async-pitr-minio/04-assert.yaml @@ -6,7 +6,5 @@ kind: PerconaServerMySQLBackup apiVersion: ps.percona.com/v1 metadata: name: async-pitr-minio-backup - finalizers: - - percona.com/delete-backup status: state: Succeeded diff --git a/e2e-tests/tests/async-pitr-minio/04-create-backup.yaml b/e2e-tests/tests/async-pitr-minio/04-create-backup.yaml index 38d73a87d..9c42eacf9 100644 --- a/e2e-tests/tests/async-pitr-minio/04-create-backup.yaml +++ b/e2e-tests/tests/async-pitr-minio/04-create-backup.yaml @@ -2,8 +2,6 @@ apiVersion: ps.percona.com/v1 kind: PerconaServerMySQLBackup metadata: name: async-pitr-minio-backup - finalizers: - - percona.com/delete-backup spec: clusterName: async-pitr-minio storageName: minio diff --git a/e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml b/e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml index 69f9161be..010ba1368 100644 --- a/e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml +++ b/e2e-tests/tests/async-pitr-minio/11-write-and-backup.yaml @@ -18,7 +18,6 @@ commands: | yq eval '.apiVersion = "ps.percona.com/v1"' - \ | yq eval '.kind = "PerconaServerMySQLBackup"' - \ | yq eval '.metadata.name = "async-pitr-minio-backup-2"' - \ - | yq eval '.metadata.finalizers[0] = "percona.com/delete-backup"' - \ | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ | yq eval '.spec.storageName = "minio"' - \ | kubectl apply -n "${NAMESPACE}" -f - diff --git a/e2e-tests/tests/gr-pitr-minio/04-assert.yaml b/e2e-tests/tests/gr-pitr-minio/04-assert.yaml index e23e05955..1f80a179f 100644 --- a/e2e-tests/tests/gr-pitr-minio/04-assert.yaml +++ b/e2e-tests/tests/gr-pitr-minio/04-assert.yaml @@ -6,7 +6,5 @@ kind: PerconaServerMySQLBackup apiVersion: ps.percona.com/v1 metadata: name: gr-pitr-minio-backup - finalizers: - - percona.com/delete-backup status: state: Succeeded diff --git a/e2e-tests/tests/gr-pitr-minio/04-create-backup.yaml b/e2e-tests/tests/gr-pitr-minio/04-create-backup.yaml index 80aa9b5d1..e4b69900e 100644 --- a/e2e-tests/tests/gr-pitr-minio/04-create-backup.yaml +++ b/e2e-tests/tests/gr-pitr-minio/04-create-backup.yaml @@ -2,8 +2,6 @@ apiVersion: ps.percona.com/v1 kind: PerconaServerMySQLBackup metadata: name: gr-pitr-minio-backup - finalizers: - - percona.com/delete-backup spec: clusterName: gr-pitr-minio storageName: minio diff --git a/e2e-tests/tests/gr-pitr-minio/11-write-and-backup.yaml b/e2e-tests/tests/gr-pitr-minio/11-write-and-backup.yaml index a25dd6dce..b3dccd2d3 100644 --- a/e2e-tests/tests/gr-pitr-minio/11-write-and-backup.yaml +++ b/e2e-tests/tests/gr-pitr-minio/11-write-and-backup.yaml @@ -18,7 +18,6 @@ commands: | yq eval '.apiVersion = "ps.percona.com/v1"' - \ | yq eval '.kind = "PerconaServerMySQLBackup"' - \ | yq eval '.metadata.name = "gr-pitr-minio-backup-2"' - \ - | yq eval '.metadata.finalizers[0] = "percona.com/delete-backup"' - \ | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ | yq eval '.spec.storageName = "minio"' - \ - | kubectl apply -n "${NAMESPACE}" -f - \ No newline at end of file + | kubectl apply -n "${NAMESPACE}" -f - From 57342274290c54aa44a3c1c40dbd9f6fc2864f74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:31:29 +0300 Subject: [PATCH 097/102] assert sleep-forever --- pkg/pitr/pitr_test.go | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pkg/pitr/pitr_test.go b/pkg/pitr/pitr_test.go index b3dd6d9b9..30164b8ba 100644 --- a/pkg/pitr/pitr_test.go +++ b/pkg/pitr/pitr_test.go @@ -274,6 +274,58 @@ func TestRestoreJob(t *testing.T) { assert.Equal(t, []corev1.LocalObjectReference{{Name: "registry-secret"}}, job.Spec.Template.Spec.ImagePullSecrets) }, }, + "sleep forever annotation sets SLEEP_FOREVER env var": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-restore", + Namespace: "ns", + Annotations: map[string]string{"percona.com/pitr-sleep-forever": "true"}, + }, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + container := job.Spec.Template.Spec.Containers[0] + envMap := envToMap(container.Env) + assert.Equal(t, "true", envMap["SLEEP_FOREVER"]) + }, + }, + "no sleep forever annotation omits SLEEP_FOREVER env var": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore", Namespace: "ns"}, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + container := job.Spec.Template.Spec.Containers[0] + envMap := envToMap(container.Env) + assert.NotContains(t, envMap, "SLEEP_FOREVER") + }, + }, "restore container has correct env vars without pitr spec": { cluster: &apiv1.PerconaServerMySQL{ ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, From 6f6ee9f5d0cdb13e02cb8d6d6b51c0f014878aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:38:32 +0300 Subject: [PATCH 098/102] add defaults to CRD --- api/v1/perconaservermysql_types.go | 9 +++++++++ config/crd/bases/ps.percona.com_perconaservermysqls.yaml | 9 +++++++++ deploy/bundle.yaml | 9 +++++++++ deploy/crd.yaml | 9 +++++++++ deploy/cw-bundle.yaml | 9 +++++++++ 5 files changed, 45 insertions(+) diff --git a/api/v1/perconaservermysql_types.go b/api/v1/perconaservermysql_types.go index 2e1844507..ccb4a0d0b 100644 --- a/api/v1/perconaservermysql_types.go +++ b/api/v1/perconaservermysql_types.go @@ -536,24 +536,33 @@ type BinlogServerSpec struct { Storage BinlogServerStorageSpec `json:"storage,omitempty"` // The number of seconds the MySQL client library will wait to establish a connection with a remote host + // +kubebuilder:default=30 ConnectTimeout int32 `json:"connectTimeout,omitempty"` // The number of seconds the MySQL client library will wait to read data from a remote server. + // +kubebuilder:default=30 ReadTimeout int32 `json:"readTimeout,omitempty"` // The number of seconds the MySQL client library will wait to write data to a remote server. + // +kubebuilder:default=30 WriteTimeout int32 `json:"writeTimeout,omitempty"` // Specifies the server ID that the utility will be using when connecting to a remote MySQL server ServerID int32 `json:"serverId,omitempty"` // The number of seconds the utility will spend in disconnected mode between reconnection attempts. + // +kubebuilder:default=30 IdleTime int32 `json:"idleTime,omitempty"` // SSLMode specifies the SSL mode for the connection to MySQL. + // +kubebuilder:default="verify_identity" SSLMode string `json:"sslMode,omitempty"` // VerifyChecksum enables checksum verification during replication. + // +kubebuilder:default=true VerifyChecksum *bool `json:"verifyChecksum,omitempty"` // RewriteFileSize specifies the maximum binlog file size for rewrite. + // +kubebuilder:default="128M" RewriteFileSize string `json:"rewriteFileSize,omitempty"` // CheckpointSize specifies the storage checkpoint size. + // +kubebuilder:default="16M" CheckpointSize string `json:"checkpointSize,omitempty"` // CheckpointInterval specifies the storage checkpoint interval. + // +kubebuilder:default="30s" CheckpointInterval string `json:"checkpointInterval,omitempty"` PodSpec `json:",inline"` diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 50112cbcd..7aac26b5d 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -710,12 +710,15 @@ spec: type: string type: object checkpointInterval: + default: 30s type: string checkpointSize: + default: 16M type: string configuration: type: string connectTimeout: + default: 30 format: int32 type: integer containerSecurityContext: @@ -902,6 +905,7 @@ spec: format: int64 type: integer idleTime: + default: 30 format: int32 type: integer image: @@ -1213,6 +1217,7 @@ spec: priorityClassName: type: string readTimeout: + default: 30 format: int32 type: integer readinessProbe: @@ -1330,6 +1335,7 @@ spec: type: object type: object rewriteFileSize: + default: 128M type: string runtimeClassName: type: string @@ -1344,6 +1350,7 @@ spec: format: int32 type: integer sslMode: + default: verify_identity type: string startupProbe: properties: @@ -1516,8 +1523,10 @@ spec: type: object type: array verifyChecksum: + default: true type: boolean writeTimeout: + default: 30 format: int32 type: integer type: object diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 63ac76399..980fcb021 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -3149,12 +3149,15 @@ spec: type: string type: object checkpointInterval: + default: 30s type: string checkpointSize: + default: 16M type: string configuration: type: string connectTimeout: + default: 30 format: int32 type: integer containerSecurityContext: @@ -3341,6 +3344,7 @@ spec: format: int64 type: integer idleTime: + default: 30 format: int32 type: integer image: @@ -3652,6 +3656,7 @@ spec: priorityClassName: type: string readTimeout: + default: 30 format: int32 type: integer readinessProbe: @@ -3769,6 +3774,7 @@ spec: type: object type: object rewriteFileSize: + default: 128M type: string runtimeClassName: type: string @@ -3783,6 +3789,7 @@ spec: format: int32 type: integer sslMode: + default: verify_identity type: string startupProbe: properties: @@ -3955,8 +3962,10 @@ spec: type: object type: array verifyChecksum: + default: true type: boolean writeTimeout: + default: 30 format: int32 type: integer type: object diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 93a3e3f2e..c0a728afa 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -3149,12 +3149,15 @@ spec: type: string type: object checkpointInterval: + default: 30s type: string checkpointSize: + default: 16M type: string configuration: type: string connectTimeout: + default: 30 format: int32 type: integer containerSecurityContext: @@ -3341,6 +3344,7 @@ spec: format: int64 type: integer idleTime: + default: 30 format: int32 type: integer image: @@ -3652,6 +3656,7 @@ spec: priorityClassName: type: string readTimeout: + default: 30 format: int32 type: integer readinessProbe: @@ -3769,6 +3774,7 @@ spec: type: object type: object rewriteFileSize: + default: 128M type: string runtimeClassName: type: string @@ -3783,6 +3789,7 @@ spec: format: int32 type: integer sslMode: + default: verify_identity type: string startupProbe: properties: @@ -3955,8 +3962,10 @@ spec: type: object type: array verifyChecksum: + default: true type: boolean writeTimeout: + default: 30 format: int32 type: integer type: object diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index eb809ad1b..d07344d3e 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -3149,12 +3149,15 @@ spec: type: string type: object checkpointInterval: + default: 30s type: string checkpointSize: + default: 16M type: string configuration: type: string connectTimeout: + default: 30 format: int32 type: integer containerSecurityContext: @@ -3341,6 +3344,7 @@ spec: format: int64 type: integer idleTime: + default: 30 format: int32 type: integer image: @@ -3652,6 +3656,7 @@ spec: priorityClassName: type: string readTimeout: + default: 30 format: int32 type: integer readinessProbe: @@ -3769,6 +3774,7 @@ spec: type: object type: object rewriteFileSize: + default: 128M type: string runtimeClassName: type: string @@ -3783,6 +3789,7 @@ spec: format: int32 type: integer sslMode: + default: verify_identity type: string startupProbe: properties: @@ -3955,8 +3962,10 @@ spec: type: object type: array verifyChecksum: + default: true type: boolean writeTimeout: + default: 30 format: int32 type: integer type: object From 001e66b6015d71d1961dc6c5c3e0acae41eeba2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:44:28 +0300 Subject: [PATCH 099/102] make force optional --- api/v1/perconaservermysqlrestore_types.go | 7 ++- cmd/pitr/main.go | 4 +- cmd/pitr/main_test.go | 24 +++++++ ...ercona.com_perconaservermysqlrestores.yaml | 2 + deploy/backup/restore.yaml | 1 + deploy/bundle.yaml | 2 + deploy/crd.yaml | 2 + deploy/cw-bundle.yaml | 2 + pkg/pitr/pitr.go | 6 ++ pkg/pitr/pitr_test.go | 62 +++++++++++++++++++ 10 files changed, 108 insertions(+), 4 deletions(-) diff --git a/api/v1/perconaservermysqlrestore_types.go b/api/v1/perconaservermysqlrestore_types.go index da4a7dc4e..b36ebe295 100644 --- a/api/v1/perconaservermysqlrestore_types.go +++ b/api/v1/perconaservermysqlrestore_types.go @@ -36,9 +36,10 @@ type PerconaServerMySQLRestoreSpec struct { type RestorePITRSpec struct { // +kubebuilder:validation:Enum=gtid;date - Type PITRType `json:"type"` - Date string `json:"date,omitempty"` - GTID string `json:"gtid,omitempty"` + Type PITRType `json:"type"` + Date string `json:"date,omitempty"` + GTID string `json:"gtid,omitempty"` + Force bool `json:"force,omitempty"` } type PITRType string diff --git a/cmd/pitr/main.go b/cmd/pitr/main.go index fbca1eea1..2a849d778 100644 --- a/cmd/pitr/main.go +++ b/cmd/pitr/main.go @@ -146,11 +146,13 @@ func run(ctx context.Context, newS3 newStorageFn, newDB newDatabaseFn, getSecret } mysqlArgs := []string{ - "--force", "-u", string(apiv1.UserOperator), "-h", "127.0.0.1", "-P", "33062", } + if os.Getenv("PITR_FORCE") == "true" { + mysqlArgs = append([]string{"--force"}, mysqlArgs...) + } log.Printf("applying %d binlog(s) with mysqlbinlog args: %v", len(objectKeys), mysqlbinlogArgs) diff --git a/cmd/pitr/main_test.go b/cmd/pitr/main_test.go index 8bfe771c8..dc9079753 100644 --- a/cmd/pitr/main_test.go +++ b/cmd/pitr/main_test.go @@ -95,6 +95,7 @@ func TestRun(t *testing.T) { pitrType string pitrGTID string pitrDate string + pitrForce string db *fakeDB newDB func(ctx context.Context, params db.DBParams) (Database, error) newS3 func(*fakeStorage) newStorageFn @@ -182,6 +183,17 @@ func TestRun(t *testing.T) { assert.Contains(t, call.mysqlbinlogArgs, "--exclude-gtids=aaaaaaaa-0000-0000-0000-000000000001:1-5") assert.Contains(t, call.mysqlbinlogArgs, "--include-gtids=aaaaaaaa-0000-0000-0000-000000000001:1-10") assert.NotContains(t, call.mysqlbinlogArgs, "--stop-datetime") + assert.NotContains(t, call.mysqlArgs, "--force") + }, + }, + "GTID mode with force": { + entries: defaultEntries, + pitrType: "gtid", + pitrGTID: "aaaaaaaa-0000-0000-0000-000000000001:1-10", + pitrForce: "true", + db: &fakeDB{getGTIDExecutedResult: "aaaaaaaa-0000-0000-0000-000000000001:1-5"}, + checkApply: func(t *testing.T, call applyCall) { + assert.Contains(t, call.mysqlArgs, "--force") }, }, "date mode success": { @@ -198,6 +210,17 @@ func TestRun(t *testing.T) { for _, arg := range call.mysqlbinlogArgs { assert.False(t, strings.HasPrefix(arg, "--include-gtids"), "date mode should not have --include-gtids") } + assert.NotContains(t, call.mysqlArgs, "--force") + }, + }, + "date mode with force": { + entries: defaultEntries, + pitrType: "date", + pitrDate: "2024-01-15 12:00:00", + pitrForce: "true", + db: &fakeDB{getGTIDExecutedResult: "bbbbbbbb-0000-0000-0000-000000000002:1-5"}, + checkApply: func(t *testing.T, call applyCall) { + assert.Contains(t, call.mysqlArgs, "--force") }, }, "empty GTID_EXECUTED": { @@ -239,6 +262,7 @@ func TestRun(t *testing.T) { t.Setenv("PITR_TYPE", tc.pitrType) t.Setenv("PITR_GTID", tc.pitrGTID) t.Setenv("PITR_DATE", tc.pitrDate) + t.Setenv("PITR_FORCE", tc.pitrForce) t.Setenv("S3_BUCKET", bucket) fakeDatabase := tc.db diff --git a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml index 40c1bf6bb..86267fca6 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml @@ -1180,6 +1180,8 @@ spec: properties: date: type: string + force: + type: boolean gtid: type: string type: diff --git a/deploy/backup/restore.yaml b/deploy/backup/restore.yaml index ebe26231c..d1f6480b0 100644 --- a/deploy/backup/restore.yaml +++ b/deploy/backup/restore.yaml @@ -7,6 +7,7 @@ spec: backupName: backup1 # pitr: # date: "2024-11-18T11:10:48Z" +# force: false # gtid: a3e5ff70-83e2-11ef-8e57-7a62caf7e1e3:1-36 # type: date # containerOptions: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 980fcb021..55ada54ac 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -2404,6 +2404,8 @@ spec: properties: date: type: string + force: + type: boolean gtid: type: string type: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index c0a728afa..2cea0803f 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -2404,6 +2404,8 @@ spec: properties: date: type: string + force: + type: boolean gtid: type: string type: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index d07344d3e..c54ff87f1 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -2404,6 +2404,8 @@ spec: properties: date: type: string + force: + type: boolean gtid: type: string type: diff --git a/pkg/pitr/pitr.go b/pkg/pitr/pitr.go index f084aaefd..e1f766f53 100644 --- a/pkg/pitr/pitr.go +++ b/pkg/pitr/pitr.go @@ -210,6 +210,12 @@ func restoreContainer( Value: restore.Spec.PITR.GTID, }) } + if restore.Spec.PITR.Force { + envs = append(envs, corev1.EnvVar{ + Name: "PITR_FORCE", + Value: "true", + }) + } } if binlogServer.Storage.S3 != nil { diff --git a/pkg/pitr/pitr_test.go b/pkg/pitr/pitr_test.go index 30164b8ba..a264e1f18 100644 --- a/pkg/pitr/pitr_test.go +++ b/pkg/pitr/pitr_test.go @@ -417,6 +417,68 @@ func TestRestoreJob(t *testing.T) { assert.Equal(t, "gtid", envMap["PITR_TYPE"]) assert.Equal(t, "abc123:1-100", envMap["PITR_GTID"]) assert.NotContains(t, envMap, "PITR_DATE") + assert.NotContains(t, envMap, "PITR_FORCE") + }, + }, + "restore container has PITR_FORCE env var when force is true": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLRestoreSpec{ + PITR: &apiv1.RestorePITRSpec{ + Type: apiv1.PITRDate, + Date: "2024-01-15 10:00:00", + Force: true, + }, + }, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + container := job.Spec.Template.Spec.Containers[0] + envMap := envToMap(container.Env) + assert.Equal(t, "true", envMap["PITR_FORCE"]) + }, + }, + "restore container omits PITR_FORCE env var when force is false": { + cluster: &apiv1.PerconaServerMySQL{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLSpec{ + SecretsName: "secrets", + SSLSecretName: "ssl", + Backup: &apiv1.BackupSpec{ + PiTR: apiv1.PiTRSpec{ + BinlogServer: &apiv1.BinlogServerSpec{}, + }, + }, + }, + }, + restore: &apiv1.PerconaServerMySQLRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "my-restore", Namespace: "ns"}, + Spec: apiv1.PerconaServerMySQLRestoreSpec{ + PITR: &apiv1.RestorePITRSpec{ + Type: apiv1.PITRGtid, + GTID: "abc123:1-100", + }, + }, + }, + storage: &apiv1.BackupStorageSpec{}, + initImage: "init:latest", + verify: func(t *testing.T, job *batchv1.Job) { + container := job.Spec.Template.Spec.Containers[0] + envMap := envToMap(container.Env) + assert.NotContains(t, envMap, "PITR_FORCE") }, }, "restore container has s3 env vars when binlog server has s3 storage": { From 6d717fed04747e3f6d54258d76eb6b45fb08153f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 00:49:27 +0300 Subject: [PATCH 100/102] add more validations --- api/v1/perconaservermysqlrestore_types.go | 2 ++ .../bases/ps.percona.com_perconaservermysqlrestores.yaml | 7 +++++++ deploy/bundle.yaml | 7 +++++++ deploy/crd.yaml | 7 +++++++ deploy/cw-bundle.yaml | 7 +++++++ 5 files changed, 30 insertions(+) diff --git a/api/v1/perconaservermysqlrestore_types.go b/api/v1/perconaservermysqlrestore_types.go index b36ebe295..b05099dd3 100644 --- a/api/v1/perconaservermysqlrestore_types.go +++ b/api/v1/perconaservermysqlrestore_types.go @@ -34,6 +34,8 @@ type PerconaServerMySQLRestoreSpec struct { PITR *RestorePITRSpec `json:"pitr,omitempty"` } +// +kubebuilder:validation:XValidation:rule="self.type != 'date' || (self.date != '' && self.gtid == '')",message="When type is 'date', 'date' must be set and 'gtid' must not be set" +// +kubebuilder:validation:XValidation:rule="self.type != 'gtid' || (self.gtid != '' && self.date == '')",message="When type is 'gtid', 'gtid' must be set and 'date' must not be set" type RestorePITRSpec struct { // +kubebuilder:validation:Enum=gtid;date Type PITRType `json:"type"` diff --git a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml index 86267fca6..3ce0f8c50 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml @@ -1192,6 +1192,13 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: When type is 'date', 'date' must be set and 'gtid' must + not be set + rule: self.type != 'date' || (self.date != '' && self.gtid == '') + - message: When type is 'gtid', 'gtid' must be set and 'date' must + not be set + rule: self.type != 'gtid' || (self.gtid != '' && self.date == '') required: - clusterName type: object diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 55ada54ac..6fd714fe3 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -2416,6 +2416,13 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: When type is 'date', 'date' must be set and 'gtid' must + not be set + rule: self.type != 'date' || (self.date != '' && self.gtid == '') + - message: When type is 'gtid', 'gtid' must be set and 'date' must + not be set + rule: self.type != 'gtid' || (self.gtid != '' && self.date == '') required: - clusterName type: object diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 2cea0803f..e97ce03a3 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -2416,6 +2416,13 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: When type is 'date', 'date' must be set and 'gtid' must + not be set + rule: self.type != 'date' || (self.date != '' && self.gtid == '') + - message: When type is 'gtid', 'gtid' must be set and 'date' must + not be set + rule: self.type != 'gtid' || (self.gtid != '' && self.date == '') required: - clusterName type: object diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index c54ff87f1..367f8f0d6 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -2416,6 +2416,13 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: When type is 'date', 'date' must be set and 'gtid' must + not be set + rule: self.type != 'date' || (self.date != '' && self.gtid == '') + - message: When type is 'gtid', 'gtid' must be set and 'date' must + not be set + rule: self.type != 'gtid' || (self.gtid != '' && self.date == '') required: - clusterName type: object From 14fd41373aa198851be73ce76c4e7bc591d4640f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 08:04:05 +0300 Subject: [PATCH 101/102] revert validation --- api/v1/perconaservermysqlrestore_types.go | 2 -- .../bases/ps.percona.com_perconaservermysqlrestores.yaml | 7 ------- deploy/bundle.yaml | 7 ------- deploy/crd.yaml | 7 ------- deploy/cw-bundle.yaml | 7 ------- 5 files changed, 30 deletions(-) diff --git a/api/v1/perconaservermysqlrestore_types.go b/api/v1/perconaservermysqlrestore_types.go index b05099dd3..b36ebe295 100644 --- a/api/v1/perconaservermysqlrestore_types.go +++ b/api/v1/perconaservermysqlrestore_types.go @@ -34,8 +34,6 @@ type PerconaServerMySQLRestoreSpec struct { PITR *RestorePITRSpec `json:"pitr,omitempty"` } -// +kubebuilder:validation:XValidation:rule="self.type != 'date' || (self.date != '' && self.gtid == '')",message="When type is 'date', 'date' must be set and 'gtid' must not be set" -// +kubebuilder:validation:XValidation:rule="self.type != 'gtid' || (self.gtid != '' && self.date == '')",message="When type is 'gtid', 'gtid' must be set and 'date' must not be set" type RestorePITRSpec struct { // +kubebuilder:validation:Enum=gtid;date Type PITRType `json:"type"` diff --git a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml index 3ce0f8c50..86267fca6 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqlrestores.yaml @@ -1192,13 +1192,6 @@ spec: required: - type type: object - x-kubernetes-validations: - - message: When type is 'date', 'date' must be set and 'gtid' must - not be set - rule: self.type != 'date' || (self.date != '' && self.gtid == '') - - message: When type is 'gtid', 'gtid' must be set and 'date' must - not be set - rule: self.type != 'gtid' || (self.gtid != '' && self.date == '') required: - clusterName type: object diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 6fd714fe3..55ada54ac 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -2416,13 +2416,6 @@ spec: required: - type type: object - x-kubernetes-validations: - - message: When type is 'date', 'date' must be set and 'gtid' must - not be set - rule: self.type != 'date' || (self.date != '' && self.gtid == '') - - message: When type is 'gtid', 'gtid' must be set and 'date' must - not be set - rule: self.type != 'gtid' || (self.gtid != '' && self.date == '') required: - clusterName type: object diff --git a/deploy/crd.yaml b/deploy/crd.yaml index e97ce03a3..2cea0803f 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -2416,13 +2416,6 @@ spec: required: - type type: object - x-kubernetes-validations: - - message: When type is 'date', 'date' must be set and 'gtid' must - not be set - rule: self.type != 'date' || (self.date != '' && self.gtid == '') - - message: When type is 'gtid', 'gtid' must be set and 'date' must - not be set - rule: self.type != 'gtid' || (self.gtid != '' && self.date == '') required: - clusterName type: object diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 367f8f0d6..c54ff87f1 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -2416,13 +2416,6 @@ spec: required: - type type: object - x-kubernetes-validations: - - message: When type is 'date', 'date' must be set and 'gtid' must - not be set - rule: self.type != 'date' || (self.date != '' && self.gtid == '') - - message: When type is 'gtid', 'gtid' must be set and 'date' must - not be set - rule: self.type != 'gtid' || (self.gtid != '' && self.date == '') required: - clusterName type: object From d09f043c22837a9cf1f8b1c7e74b43ac664a47e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 16 Apr 2026 11:09:17 +0300 Subject: [PATCH 102/102] use force after failover --- e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml b/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml index 382e032c1..fe2e9760e 100644 --- a/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml +++ b/e2e-tests/tests/gr-pitr-minio/12-write-and-restore.yaml @@ -25,6 +25,7 @@ commands: | yq eval '.metadata.name = "gr-pitr-minio-restore-post-failover"' - \ | yq eval ".spec.clusterName = \"${cluster_name}\"" - \ | yq eval '.spec.backupName = "gr-pitr-minio-backup-2"' - \ + | yq eval '.spec.pitr.force = true' - \ | yq eval '.spec.pitr.type = "date"' - \ | yq eval ".spec.pitr.date = \"${PITR_DATE_POST}\"" - \ | kubectl apply -n "${NAMESPACE}" -f -