feat(backup): integrate HCPEtcdBackup lifecycle into OADP backup flow

Add etcdSnapshot backup method that creates and monitors HCPEtcdBackup CRs
during Velero backup. When etcdBackupMethod=etcdSnapshot is configured in the
plugin ConfigMap, the plugin:

- Creates an HCPEtcdBackup CR in the HCP namespace using BSL storage config
- Copies BSL credentials to the HO namespace (remapping key for controller)
- Polls the CR until backup completes or fails
- Excludes etcd pods and PVCs from Velero backup (no CSI/FS backup needed)
- Stores the etcd snapshot alongside the Velero backup data in the BSL

The default method remains volumeSnapshot (unchanged behavior).

Also cleans up dead config parameters (readoptNodes, managedServices,
awsRegenPrivateLink) and registers apiextensionsv1 in the scheme for
CRD existence checks.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Signed-off-by: Juan Manuel Parrilla Madrid <jparrill@redhat.com>

Author: jparrill

pkg/common/scheme.go

@@ -7,6 +7,7 @@ import (
 	veleroapiv1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	veleroapiv2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -35,6 +36,9 @@ func init() {
 	if err := hive.AddToScheme(CustomScheme); err != nil {
 		errs = append(errs, err)
 	}
+	if err := apiextensionsv1.AddToScheme(CustomScheme); err != nil {
+		errs = append(errs, err)
+	}
 
 	if len(errs) > 0 {
 		panic(errs)

pkg/common/types.go

@@ -16,6 +16,9 @@ const (
 
 	// Integration with Hypershift, more info here: https://github.com/openshift/hypershift/pull/6195
 	HostedClusterRestoredFromBackupAnnotation string = "hypershift.openshift.io/restored-from-backup"
+	// Etcd snapshot URL annotation: set during backup so the restore plugin can read it
+	// (Velero strips status from items during restore, so we persist it as an annotation)
+	EtcdSnapshotURLAnnotation string = "hypershift.openshift.io/etcd-snapshot-url"
 
 	// hypershift/cluster-api kinds
 	HostedClusterKind         string = "HostedCluster"
@@ -25,6 +28,29 @@ const (
 	PersistentVolumeClaimKind string = "PersistentVolumeClaim"
 	ClusterDeploymentKind     string = "ClusterDeployment"
 	DataVolumeKind            string = "DataVolume"
+	HCPEtcdBackupKind         string = "HCPEtcdBackup"
+
+	// Default HyperShift Operator namespace
+	DefaultHONamespace string = "hypershift"
+	// ConfigMap key to override the HO namespace
+	ConfigKeyHONamespace string = "hoNamespace"
+
+	// Etcd backup method configuration
+	ConfigKeyEtcdBackupMethod    string = "etcdBackupMethod"
+	EtcdBackupMethodVolume       string = "volumeSnapshot"
+	EtcdBackupMethodEtcdSnapshot string = "etcdSnapshot"
+
+	// Velero annotation to exclude specific volumes from backup
+	BackupVolumesExcludesAnnotation string = "backup.velero.io/backup-volumes-excludes"
+	// Etcd data volume name in the StatefulSet pod
+	EtcdDataVolumeName string = "data"
+	// Etcd PVC name prefix (StatefulSet pattern: {volumeName}-{stsName}-{index})
+	EtcdPVCPrefix string = "data-etcd-"
+
+	// TODO(CNTRLPLANE-2685): Remove these local constants once openshift/hypershift#8139 is merged
+	// and the vendor is updated. These must match the values used by the HCPEtcdBackup controller.
+	BackupInProgressReason string = "BackupInProgress"
+	BackupRejectedReason   string = "BackupRejected"
 )
 
 var (

pkg/common/utils.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -163,6 +164,32 @@ func GetHCPNamespace(name, namespace string) string {
 	return fmt.Sprintf("%s-%s", namespace, name)
 }
 
+// GetHostedCluster finds the HostedCluster that owns the HCP by deriving
+// its namespace and name from the HCP namespace convention: {hc-namespace}-{hc-name}.
+func GetHostedCluster(ctx context.Context, c crclient.Client, includedNamespaces []string, hcpNamespace string) (*hyperv1.HostedCluster, error) {
+	var errs []error
+	for _, ns := range includedNamespaces {
+		if ns == hcpNamespace {
+			continue
+		}
+		hcList := &hyperv1.HostedClusterList{}
+		if err := c.List(ctx, hcList, crclient.InNamespace(ns)); err != nil {
+			errs = append(errs, fmt.Errorf("list HostedClusters in namespace %s: %w", ns, err))
+			continue
+		}
+		for i := range hcList.Items {
+			hc := &hcList.Items[i]
+			if GetHCPNamespace(hc.Name, hc.Namespace) == hcpNamespace {
+				return hc, nil
+			}
+		}
+	}
+	if len(errs) > 0 {
+		return nil, errors.Join(errs...)
+	}
+	return nil, nil
+}
+
 // ShouldEndPluginExecution checks if the plugin should end execution by verifying if the required
 // Hypershift resources (HostedControlPlane and HostedCluster) exist in the cluster.
 // Returns true if the plugin should end execution (i.e., if this is not a Hypershift cluster).

pkg/common/utils_test.go

@@ -573,6 +573,34 @@ func TestShouldEndPluginExecution(t *testing.T) {
 	}
 }
 
+func TestGetHostedCluster(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = hyperv1.AddToScheme(scheme)
+
+	t.Run("When GetHostedCluster runs with a HostedCluster matching HCP namespace, It Should return that cluster", func(t *testing.T) {
+		g := NewWithT(t)
+		hc := &hyperv1.HostedCluster{
+			ObjectMeta: metav1.ObjectMeta{Name: "my-cluster", Namespace: "clusters"},
+		}
+		c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(hc).Build()
+
+		result, err := GetHostedCluster(context.TODO(), c, []string{"clusters", "clusters-my-cluster"}, "clusters-my-cluster")
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(result).NotTo(BeNil())
+		g.Expect(result.Name).To(Equal("my-cluster"))
+		g.Expect(result.Namespace).To(Equal("clusters"))
+	})
+
+	t.Run("When GetHostedCluster runs with no HostedClusters in client, It Should return nil", func(t *testing.T) {
+		g := NewWithT(t)
+		c := fake.NewClientBuilder().WithScheme(scheme).Build()
+
+		result, err := GetHostedCluster(context.TODO(), c, []string{"clusters", "clusters-my-cluster"}, "clusters-my-cluster")
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(result).To(BeNil())
+	})
+}
+
 func TestCRDExists(t *testing.T) {
 	scheme := runtime.NewScheme()
 	_ = hyperv1.AddToScheme(scheme)

pkg/core/backup.go

@@ -8,6 +8,7 @@ import (
 	common "github.com/openshift/hypershift-oadp-plugin/pkg/common"
 	plugtypes "github.com/openshift/hypershift-oadp-plugin/pkg/core/types"
 	validation "github.com/openshift/hypershift-oadp-plugin/pkg/core/validation"
+	"github.com/openshift/hypershift-oadp-plugin/pkg/etcdbackup"
 	"github.com/openshift/hypershift-oadp-plugin/pkg/platform/agent"
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	"github.com/sirupsen/logrus"
@@ -31,6 +32,12 @@ type BackupPlugin struct {
 	validator validation.BackupValidator
 	hcp       *hyperv1.HostedControlPlane
 	*plugtypes.BackupOptions
+
+	// Etcd backup orchestration
+	etcdOrchestrator  *etcdbackup.Orchestrator
+	hoNamespace       string
+	etcdBackupMethod  string
+	etcdSnapshotURL   string // populated after HCPEtcdBackup completes
 }
 
 // NewBackupPlugin instantiates BackupPlugin.
@@ -71,12 +78,27 @@ func NewBackupPlugin(logger logrus.FieldLogger) (*BackupPlugin, error) {
 		Client: client,
 	}
 
+	hoNamespace := common.DefaultHONamespace
+	if v, ok := pluginConfig.Data[common.ConfigKeyHONamespace]; ok && v != "" {
+		hoNamespace = v
+	}
+
+	etcdBackupMethod := common.EtcdBackupMethodVolume
+	if v, ok := pluginConfig.Data[common.ConfigKeyEtcdBackupMethod]; ok && v != "" {
+		etcdBackupMethod = v
+	}
+	if etcdBackupMethod != common.EtcdBackupMethodVolume && etcdBackupMethod != common.EtcdBackupMethodEtcdSnapshot {
+		return nil, fmt.Errorf("invalid etcdBackupMethod %q: must be %q or %q", etcdBackupMethod, common.EtcdBackupMethodVolume, common.EtcdBackupMethodEtcdSnapshot)
+	}
+
 	bp := &BackupPlugin{
-		log:       logger,
-		client:    client,
-		config:    pluginConfig.Data,
-		ctx:       ctx,
-		validator: validator,
+		log:              logger,
+		client:           client,
+		config:           pluginConfig.Data,
+		ctx:              ctx,
+		validator:        validator,
+		hoNamespace:      hoNamespace,
+		etcdBackupMethod: etcdBackupMethod,
 	}
 
 	if bp.BackupOptions, err = bp.validator.ValidatePluginConfig(bp.config); err != nil {
@@ -119,7 +141,6 @@ func (p *BackupPlugin) Execute(item runtime.Unstructured, backup *velerov1.Backu
 			}
 			return nil, nil, fmt.Errorf("error getting HCP namespace: %v", err)
 		}
-
 	}
 
 	kind := item.GetObjectKind().GroupVersionKind().Kind
@@ -134,6 +155,16 @@ func (p *BackupPlugin) Execute(item runtime.Unstructured, backup *velerov1.Backu
 			return nil, nil, fmt.Errorf("error checking platform configuration: %v", err)
 		}
 
+		// Etcd backup: create after validation, wait for completion
+		if p.etcdBackupMethod == common.EtcdBackupMethodEtcdSnapshot {
+			if err := p.createEtcdBackup(ctx, backup); err != nil {
+				return nil, nil, fmt.Errorf("error creating HCPEtcdBackup: %v", err)
+			}
+		}
+		if err := p.waitForEtcdBackupCompletion(ctx); err != nil {
+			return nil, nil, err
+		}
+
 	case kind == common.HostedClusterKind:
 		metadata, err := meta.Accessor(item)
 		if err != nil {
@@ -142,16 +173,53 @@ func (p *BackupPlugin) Execute(item runtime.Unstructured, backup *velerov1.Backu
 		common.AddAnnotation(metadata, common.HostedClusterRestoredFromBackupAnnotation, "")
 		p.log.Infof("Added restore annotation to HostedCluster %s", metadata.GetName())
 
-	case kind == "Pod":
-		// In case of FSBackup, we need to add the label to the pod
-		if backup.Spec.DefaultVolumesToFsBackup != nil && !*backup.Spec.DefaultVolumesToFsBackup {
-			metadata, err := meta.Accessor(item)
-			if err != nil {
-				return nil, nil, fmt.Errorf("error getting metadata accessor: %v", err)
+		// Etcd backup: create if not yet created (HC may arrive before HCP),
+		// wait for completion, and inject snapshotURL into the HC item.
+		// Velero captures the item as-is from the API server before the HCPEtcdBackup
+		// controller updates the HC status with lastSuccessfulEtcdBackupURL.
+		// We must inject it here so the backed-up HC contains the URL for restore.
+		if p.etcdBackupMethod == common.EtcdBackupMethodEtcdSnapshot {
+			if err := p.createEtcdBackup(ctx, backup); err != nil {
+				return nil, nil, fmt.Errorf("error creating HCPEtcdBackup: %v", err)
 			}
+		}
+		if err := p.waitForEtcdBackupCompletion(ctx); err != nil {
+			return nil, nil, err
+		}
+		if p.etcdSnapshotURL != "" {
+			// Persist as annotation so the restore plugin can read it
+			// (Velero strips status from items during restore)
+			common.AddAnnotation(metadata, common.EtcdSnapshotURLAnnotation, p.etcdSnapshotURL)
+			p.log.Infof("Added etcd snapshot URL annotation to HostedCluster %s: %s", metadata.GetName(), p.etcdSnapshotURL)
 
-			if strings.Contains(metadata.GetName(), "etcd-") {
-				common.AddLabel(metadata, common.FSBackupLabelName, "true")
+			unstructuredContent := item.UnstructuredContent()
+			status, ok := unstructuredContent["status"].(map[string]interface{})
+			if !ok {
+				status = map[string]interface{}{}
+				unstructuredContent["status"] = status
+			}
+			status["lastSuccessfulEtcdBackupURL"] = p.etcdSnapshotURL
+			item.SetUnstructuredContent(unstructuredContent)
+			p.log.Infof("Injected lastSuccessfulEtcdBackupURL into HostedCluster %s: %s", metadata.GetName(), p.etcdSnapshotURL)
+		}
+
+	case kind == "Pod":
+		metadata, err := meta.Accessor(item)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error getting metadata accessor: %v", err)
+		}
+
+		if strings.Contains(metadata.GetName(), "etcd-") {
+			switch p.etcdBackupMethod {
+			case common.EtcdBackupMethodEtcdSnapshot:
+				// Skip etcd pods entirely, snapshot is handled by HCPEtcdBackup.
+				// This prevents both FSBackup and CSI VolumeSnapshots of etcd volumes.
+				p.log.Infof("Skipping etcd pod %s from backup (using etcdSnapshot method)", metadata.GetName())
+				return nil, nil, nil
+			case common.EtcdBackupMethodVolume:
+				if backup.Spec.DefaultVolumesToFsBackup != nil && !*backup.Spec.DefaultVolumesToFsBackup {
+					common.AddLabel(metadata, common.FSBackupLabelName, "true")
+				}
 			}
 		}
 
@@ -172,7 +240,91 @@ func (p *BackupPlugin) Execute(item runtime.Unstructured, backup *velerov1.Backu
 		if _, exists := labels[common.KubevirtRHCOSLabel]; exists {
 			return nil, nil, nil
 		}
+
+		// Exclude etcd data PVCs when using etcdSnapshot method.
+		// PVC names follow the StatefulSet pattern: data-etcd-{index}
+		if kind == common.PersistentVolumeClaimKind &&
+			strings.HasPrefix(metadata.GetName(), common.EtcdPVCPrefix) &&
+			p.etcdBackupMethod == common.EtcdBackupMethodEtcdSnapshot {
+			p.log.Infof("Excluding etcd PVC %s from backup (using etcdSnapshot method)", metadata.GetName())
+			return nil, nil, nil
+		}
 	}
 
 	return item, nil, nil
 }
+
+// createEtcdBackup creates an HCPEtcdBackup CR in the HCP namespace.
+// It is idempotent: if the orchestrator already created a backup, it returns immediately.
+// Requires the HCPEtcdBackup CRD to exist in the cluster (safenet check).
+func (p *BackupPlugin) createEtcdBackup(ctx context.Context, backup *velerov1.Backup) error {
+	// Already created by a previous Execute() call
+	if p.etcdOrchestrator != nil && p.etcdOrchestrator.IsCreated() {
+		return nil
+	}
+
+	crdExists, err := common.CRDExists(ctx, "hcpetcdbackups.hypershift.openshift.io", p.client)
+	if err != nil {
+		return fmt.Errorf("failed to check for HCPEtcdBackup CRD: %w", err)
+	}
+	if !crdExists {
+		return fmt.Errorf("etcdBackupMethod is %q but HCPEtcdBackup CRD not found in the cluster", common.EtcdBackupMethodEtcdSnapshot)
+	}
+
+	oadpNS, err := common.GetCurrentNamespace()
+	if err != nil {
+		return fmt.Errorf("failed to get OADP namespace: %w", err)
+	}
+
+	p.etcdOrchestrator = etcdbackup.NewOrchestrator(p.log, p.client, p.hoNamespace, oadpNS)
+
+	// Fetch the HostedCluster for encryption config
+	hc, err := common.GetHostedCluster(ctx, p.client, backup.Spec.IncludedNamespaces, p.hcp.Namespace)
+	if err != nil {
+		p.log.Warnf("Could not find HostedCluster for encryption config: %v", err)
+	}
+
+	if err := p.etcdOrchestrator.CreateEtcdBackup(ctx, backup, p.hcp.Namespace, hc); err != nil {
+		if cleanupErr := p.etcdOrchestrator.CleanupCredentialSecret(ctx); cleanupErr != nil {
+			p.log.Warnf("Failed to cleanup credential Secret after create error: %v", cleanupErr)
+		}
+		return err
+	}
+
+	if err := p.etcdOrchestrator.VerifyInProgress(ctx); err != nil {
+		if cleanupErr := p.etcdOrchestrator.CleanupCredentialSecret(ctx); cleanupErr != nil {
+			p.log.Warnf("Failed to cleanup credential Secret after verify error: %v", cleanupErr)
+		}
+		return err
+	}
+
+	return nil
+}
+
+// waitForEtcdBackupCompletion waits for the HCPEtcdBackup to finish and cleans up
+// the copied credential Secret. Caches the snapshotURL on the plugin struct so it
+// is available regardless of item processing order (HC before HCP or vice versa).
+// It is a no-op if no etcd backup was created.
+func (p *BackupPlugin) waitForEtcdBackupCompletion(ctx context.Context) error {
+	if p.etcdOrchestrator == nil || !p.etcdOrchestrator.IsCreated() {
+		return nil
+	}
+
+	// Already completed in a previous Execute() call
+	if p.etcdSnapshotURL != "" {
+		return nil
+	}
+
+	snapshotURL, err := p.etcdOrchestrator.WaitForCompletion(ctx)
+	if err != nil {
+		return fmt.Errorf("HCPEtcdBackup failed: %v", err)
+	}
+	p.etcdSnapshotURL = snapshotURL
+	p.log.Infof("HCPEtcdBackup completed, snapshotURL: %s", snapshotURL)
+
+	if cleanupErr := p.etcdOrchestrator.CleanupCredentialSecret(ctx); cleanupErr != nil {
+		p.log.Warnf("Failed to cleanup etcd backup credential Secret: %v", cleanupErr)
+	}
+
+	return nil
+}

pkg/core/types/types.go

@@ -3,6 +3,7 @@ package types
 var (
 	BackupCommonResources = []string{
 		"hostedclusters", "hostedcluster", "hostedcontrolplanes", "hostedcontrolplane", "nodepools", "nodepool",
+		"hcpetcdbackups", "hcpetcdbackup",
 		"secrets", "secret", "configmaps", "configmap", "persistentvolumes", "persistentvolume", "persistentvolumeclaims", "persistentvolumeclaim", "pods", "pod", "statefulsets", "statefulset", "deployments", "deployment",
 		"clusters", "cluster", "machines", "machine", "machinedeployments", "machinedeployment", "machinesets", "machineset",
 		"serviceaccounts", "serviceaccount", "roles", "role", "rolebindings", "rolebinding",
@@ -20,17 +21,9 @@ var (
 type BackupOptions struct {
 	// Migration is a flag to indicate if the backup is for migration purposes.
 	Migration bool
-	// Readopt Nodes is a flag to indicate if the nodes should be reprovisioned or not during restore.
-	ReadoptNodes bool
-	// ManagedServices is a flag to indicate if the backup is done for ManagedServices like ROSA, ARO, etc.
-	ManagedServices bool
 }
 
 type RestoreOptions struct {
 	// Migration is a flag to indicate if the backup is for migration purposes.
 	Migration bool
-	// Readopt Nodes is a flag to indicate if the nodes should be reprovisioned or not during restore.
-	ReadoptNodes bool
-	// ManagedServices is a flag to indicate if the backup is done for ManagedServices like ROSA, ARO, etc.
-	ManagedServices bool
 }