(fix) catalog deletion resilience support

camilamacedo86 · camilamacedo86 · commit 773f156c6425 · 2026-01-12T10:57:13.000Z
Enables installed extensions to continue working when their source
catalog becomes unavailable or is deleted. When resolution fails due
to catalog unavailability, the operator now continues reconciling with
the currently installed bundle instead of failing.

Changes:
- Resolution falls back to installed bundle when catalog unavailable
- Unpacking skipped when maintaining current installed state
- Helm and Boxcutter appliers handle nil contentFS gracefully
- Version upgrades properly blocked without catalog access

This ensures workloads remain stable and operational even when the
catalog they were installed from is temporarily unavailable or deleted,
while appropriately preventing version changes that require catalog access.
diff --git a/internal/operator-controller/applier/boxcutter.go b/internal/operator-controller/applier/boxcutter.go
@@ -267,7 +267,6 @@ func (m *BoxcutterStorageMigrator) Migrate(ctx context.Context, ext *ocv1.Cluste
 	}
 
 	// Set initial status on the migrated revision to mark it as succeeded.
-	//
 	// The revision must have a Succeeded=True status condition immediately after creation.
 	//
 	// A revision is only considered "Installed" (vs "RollingOut") when it has this condition.
@@ -330,22 +329,35 @@ func (bc *Boxcutter) createOrUpdate(ctx context.Context, obj client.Object) erro
 }
 
 func (bc *Boxcutter) apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExtension, objectLabels, revisionAnnotations map[string]string) (bool, string, error) {
-	// Generate desired revision
-	desiredRevision, err := bc.RevisionGenerator.GenerateRevision(ctx, contentFS, ext, objectLabels, revisionAnnotations)
+	// List all existing revisions
+	existingRevisions, err := bc.getExistingRevisions(ctx, ext.GetName())
 	if err != nil {
 		return false, "", err
 	}
 
-	if err := controllerutil.SetControllerReference(ext, desiredRevision, bc.Scheme); err != nil {
-		return false, "", fmt.Errorf("set ownerref: %w", err)
+	// If contentFS is nil, we're maintaining the current state without catalog access.
+	// In this case, we should use the existing installed revision without generating a new one.
+	if contentFS == nil {
+		if len(existingRevisions) == 0 {
+			return false, "", fmt.Errorf("cannot maintain workload: no catalog content available and no previously installed revision found")
+		}
+		// Use the most recent revision and rely on its existing controller loop (don't create a new one).
+		// Returning true here signals that the rollout has succeeded using the current revision; the
+		// ClusterExtensionRevision controller will continue to reconcile and maintain the resources
+		// independently of this apply call.
+		return true, "", nil
 	}
 
-	// List all existing revisions
-	existingRevisions, err := bc.getExistingRevisions(ctx, ext.GetName())
+	// Generate desired revision
+	desiredRevision, err := bc.RevisionGenerator.GenerateRevision(ctx, contentFS, ext, objectLabels, revisionAnnotations)
 	if err != nil {
 		return false, "", err
 	}
 
+	if err := controllerutil.SetControllerReference(ext, desiredRevision, bc.Scheme); err != nil {
+		return false, "", fmt.Errorf("set ownerref: %w", err)
+	}
+
 	currentRevision := &ocv1.ClusterExtensionRevision{}
 	state := StateNeedsInstall
 	// check if we can update the current revision.
diff --git a/internal/operator-controller/applier/helm.go b/internal/operator-controller/applier/helm.go
@@ -103,6 +103,16 @@ func (h *Helm) runPreAuthorizationChecks(ctx context.Context, ext *ocv1.ClusterE
 }
 
 func (h *Helm) Apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExtension, objectLabels map[string]string, storageLabels map[string]string) (bool, string, error) {
+	// If contentFS is nil, we're maintaining the current state without catalog access.
+	// In this case, reconcile the existing Helm release if it exists.
+	if contentFS == nil {
+		ac, err := h.ActionClientGetter.ActionClientFor(ctx, ext)
+		if err != nil {
+			return false, "", err
+		}
+		return h.reconcileExistingRelease(ctx, ac, ext)
+	}
+
 	chrt, err := h.buildHelmChart(contentFS, ext)
 	if err != nil {
 		return false, "", err
@@ -197,6 +207,45 @@ func (h *Helm) Apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExte
 	return true, "", nil
 }
 
+// reconcileExistingRelease reconciles an existing Helm release without catalog access.
+// This is used when the catalog is unavailable but we need to maintain the current installation.
+// It reconciles the release and sets up watchers to ensure resources are maintained.
+func (h *Helm) reconcileExistingRelease(ctx context.Context, ac helmclient.ActionInterface, ext *ocv1.ClusterExtension) (bool, string, error) {
+	rel, err := ac.Get(ext.GetName())
+	if errors.Is(err, driver.ErrReleaseNotFound) {
+		return false, "", fmt.Errorf("cannot maintain workload: no catalog content available and no previously installed Helm release found")
+	}
+	if err != nil {
+		return false, "", fmt.Errorf("getting current release: %w", err)
+	}
+
+	// Reconcile the existing release to ensure resources are maintained
+	if err := ac.Reconcile(rel); err != nil {
+		// Reconcile failed - resources NOT maintained
+		// Return false (rollout failed) with error
+		return false, "", err
+	}
+
+	// At this point: Reconcile succeeded - resources ARE maintained
+	// The operations below are for setting up monitoring (watches).
+	// If they fail, the resources are still successfully reconciled and maintained,
+	// so we return true (rollout succeeded) even though monitoring setup failed.
+	relObjects, err := util.ManifestObjects(strings.NewReader(rel.Manifest), fmt.Sprintf("%s-release-manifest", rel.Name))
+	if err != nil {
+		return true, "", err
+	}
+	klog.FromContext(ctx).Info("watching managed objects")
+	cache, err := h.Manager.Get(ctx, ext)
+	if err != nil {
+		return true, "", err
+	}
+	if err := cache.Watch(ctx, h.Watcher, relObjects...); err != nil {
+		return true, "", err
+	}
+
+	return true, "", nil
+}
+
 func (h *Helm) buildHelmChart(bundleFS fs.FS, ext *ocv1.ClusterExtension) (*chart.Chart, error) {
 	if h.HelmChartProvider == nil {
 		return nil, errors.New("HelmChartProvider is nil")
diff --git a/internal/operator-controller/controllers/clusterextension_controller_test.go b/internal/operator-controller/controllers/clusterextension_controller_test.go
@@ -1611,3 +1611,189 @@ func TestGetInstalledBundleHistory(t *testing.T) {
 		}
 	}
 }
+
+// TestResolutionFallbackToInstalledBundle tests the catalog deletion resilience fallback logic
+func TestResolutionFallbackToInstalledBundle(t *testing.T) {
+	t.Run("falls back when catalog unavailable and no version change", func(t *testing.T) {
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			// Resolver fails (simulating catalog unavailable)
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				return nil, nil, nil, fmt.Errorf("catalog unavailable")
+			})
+			// Applier succeeds (resources maintained)
+			d.Applier = &MockApplier{
+				installCompleted: true,
+				installStatus:    "",
+				err:              nil,
+			}
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						Package:        "test-pkg",
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+						Image:          "test-image:1.0.0",
+					},
+				},
+			}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		// Create ClusterExtension with no version specified
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						// No version - should fall back
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// Reconcile should succeed (fallback to installed, then apply succeeds)
+		// Catalog watch will trigger reconciliation when catalog becomes available again
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Verify status shows successful reconciliation
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+
+		// Progressing should be Succeeded (apply completed successfully)
+		progCond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeProgressing)
+		require.NotNil(t, progCond)
+		require.Equal(t, metav1.ConditionTrue, progCond.Status)
+		require.Equal(t, ocv1.ReasonSucceeded, progCond.Reason)
+
+		// Installed should be True (maintaining current version)
+		instCond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeInstalled)
+		require.NotNil(t, instCond)
+		require.Equal(t, metav1.ConditionTrue, instCond.Status)
+		require.Equal(t, ocv1.ReasonSucceeded, instCond.Reason)
+	})
+
+	t.Run("fails when version upgrade requested without catalog", func(t *testing.T) {
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				return nil, nil, nil, fmt.Errorf("catalog unavailable")
+			})
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+					},
+				},
+			}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		// Create ClusterExtension requesting version upgrade
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						Version:     "1.0.1", // Requesting upgrade
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// Reconcile should fail (can't upgrade without catalog)
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.Error(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Verify status shows Retrying
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		cond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeProgressing)
+		require.NotNil(t, cond)
+		require.Equal(t, metav1.ConditionTrue, cond.Status)
+		require.Equal(t, ocv1.ReasonRetrying, cond.Reason)
+	})
+
+	t.Run("auto-updates when catalog becomes available after fallback", func(t *testing.T) {
+		resolveAttempt := 0
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			// First attempt: catalog unavailable, then becomes available
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				resolveAttempt++
+				if resolveAttempt == 1 {
+					// First reconcile: catalog unavailable
+					return nil, nil, nil, fmt.Errorf("catalog temporarily unavailable")
+				}
+				// Second reconcile (triggered by catalog watch): catalog available with new version
+				v := bundle.VersionRelease{Version: bsemver.MustParse("2.0.0")}
+				return &declcfg.Bundle{
+					Name:    "test.2.0.0",
+					Package: "test-pkg",
+					Image:   "test-image:2.0.0",
+				}, &v, nil, nil
+			})
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						Package:        "test-pkg",
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+						Image:          "test-image:1.0.0",
+					},
+				},
+			}
+			d.ImagePuller = &imageutil.MockPuller{ImageFS: fstest.MapFS{}}
+			d.Applier = &MockApplier{installCompleted: true}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						// No version - auto-update to latest
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// First reconcile: catalog unavailable, falls back to v1.0.0
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		require.Equal(t, "1.0.0", ext.Status.Install.Bundle.Version)
+
+		// Second reconcile: simulating catalog watch trigger, catalog now available with v2.0.0
+		res, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Should have upgraded to v2.0.0
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		require.Equal(t, "2.0.0", ext.Status.Install.Bundle.Version)
+
+		// Verify resolution was attempted twice (fallback, then success)
+		require.Equal(t, 2, resolveAttempt)
+	})
+}
diff --git a/internal/operator-controller/controllers/clusterextension_reconcile_steps.go b/internal/operator-controller/controllers/clusterextension_reconcile_steps.go
@@ -95,11 +95,7 @@ func ResolveBundle(r resolve.Resolver) ReconcileStepFunc {
 			}
 			resolvedBundle, resolvedBundleVersion, resolvedDeprecation, err := r.Resolve(ctx, ext, bm)
 			if err != nil {
-				// Note: We don't distinguish between resolution-specific errors and generic errors
-				setStatusProgressing(ext, err)
-				setInstalledStatusFromRevisionStates(ext, state.revisionStates)
-				ensureAllConditionsWithReason(ext, ocv1.ReasonFailed, err.Error())
-				return nil, err
+				return handleResolutionError(ctx, state, ext, err)
 			}
 
 			// set deprecation status after _successful_ resolution
@@ -134,9 +130,73 @@ func ResolveBundle(r resolve.Resolver) ReconcileStepFunc {
 	}
 }
 
+// handleResolutionError handles the case when bundle resolution fails.
+// If a bundle is already installed and the spec isn't requesting a version change,
+// we fall back to using the installed bundle to maintain the current state.
+// This enables workload resilience when the catalog becomes unavailable.
+// However, if the spec explicitly requests a different version, we must fail and retry.
+func handleResolutionError(ctx context.Context, state *reconcileState, ext *ocv1.ClusterExtension, err error) (*ctrl.Result, error) {
+	l := log.FromContext(ctx)
+
+	// If we have an installed bundle, check if we can fall back to it
+	if state.revisionStates.Installed != nil {
+		// Check if the spec is requesting a specific version that differs from installed
+		specVersion := ""
+		if ext.Spec.Source.Catalog != nil {
+			specVersion = ext.Spec.Source.Catalog.Version
+		}
+		installedVersion := state.revisionStates.Installed.Version
+
+		// If spec requests a different version, we cannot fall back - must fail and retry
+		if specVersion != "" && specVersion != installedVersion {
+			l.Info("resolution failed and spec requests version change - cannot fall back",
+				"error", err,
+				"requestedVersion", specVersion,
+				"installedVersion", installedVersion)
+			setStatusProgressing(ext, err)
+			setInstalledStatusFromRevisionStates(ext, state.revisionStates)
+			ensureAllConditionsWithReason(ext, ocv1.ReasonRetrying, err.Error())
+			return nil, err
+		}
+
+		// No version change requested - fall back to maintain current state
+		// Continue reconciliation to maintain resources. The controller is configured to watch
+		// ClusterCatalog resources (see SetupWithManager) and will enqueue reconcile requests
+		// for this ClusterExtension when catalog updates occur. Reconciliation will also be
+		// triggered periodically via the controller's resync period.
+		l.Info("resolution failed but continuing with installed bundle", "error", err, "installedBundle", state.revisionStates.Installed.BundleMetadata)
+		// Set Progressing condition to indicate we're operating in degraded mode (retrying resolution)
+		setStatusProgressing(ext, err)
+		setInstalledStatusFromRevisionStates(ext, state.revisionStates)
+		state.resolvedRevisionMetadata = state.revisionStates.Installed
+		// Return no error so the Apply step runs to maintain resources while we wait for
+		// a new reconcile triggered by catalog watch events or periodic resync.
+		return nil, nil
+	}
+
+	// No installed bundle and resolution failed - cannot proceed
+	setStatusProgressing(ext, err)
+	setInstalledStatusFromRevisionStates(ext, state.revisionStates)
+	ensureAllConditionsWithReason(ext, ocv1.ReasonFailed, err.Error())
+	return nil, err
+}
+
 func UnpackBundle(i imageutil.Puller, cache imageutil.Cache) ReconcileStepFunc {
 	return func(ctx context.Context, state *reconcileState, ext *ocv1.ClusterExtension) (*ctrl.Result, error) {
 		l := log.FromContext(ctx)
+
+		// Skip unpacking if we're using an already-installed bundle
+		// (e.g., when catalog is unavailable but we're maintaining current state)
+		if state.revisionStates.Installed != nil &&
+			state.resolvedRevisionMetadata.Name == state.revisionStates.Installed.Name &&
+			state.resolvedRevisionMetadata.Version == state.revisionStates.Installed.Version {
+			l.Info("skipping unpack - using installed bundle content")
+			// imageFS will remain nil; applier implementations MUST handle nil imageFS by using
+			// existing installed content. See Helm.reconcileExistingRelease() and Boxcutter.apply()
+			// for nil contentFS handling.
+			return nil, nil
+		}
+
 		l.Info("unpacking resolved bundle")
 		imageFS, _, _, err := i.Pull(ctx, ext.GetName(), state.resolvedRevisionMetadata.Image, cache)
 		if err != nil {
