(fix) catalog deletion resilience support

Enables installed extensions to continue working when their source
catalog becomes unavailable or is deleted. When resolution fails due
to catalog unavailability, the operator now continues reconciling with
the currently installed bundle instead of failing.

Changes:
- Resolution falls back to installed bundle when catalog unavailable
- Unpacking skipped when maintaining current installed state
- Helm and Boxcutter appliers handle nil contentFS gracefully
- Version upgrades properly blocked without catalog access

This ensures workloads remain stable and operational even when the
catalog they were installed from is temporarily unavailable or deleted,
while appropriately preventing version changes that require catalog access.

Author: camilamacedo86

cmd/operator-controller/main.go

@@ -626,7 +626,7 @@ func (c *boxcutterReconcilerConfigurator) Configure(ceReconciler *controllers.Cl
 		controllers.HandleFinalizers(c.finalizers),
 		controllers.MigrateStorage(storageMigrator),
 		controllers.RetrieveRevisionStates(revisionStatesGetter),
-		controllers.ResolveBundle(c.resolver),
+		controllers.ResolveBundle(c.resolver, c.mgr.GetClient()),
 		controllers.UnpackBundle(c.imagePuller, c.imageCache),
 		controllers.ApplyBundleWithBoxcutter(appl),
 	}
@@ -737,7 +737,7 @@ func (c *helmReconcilerConfigurator) Configure(ceReconciler *controllers.Cluster
 	ceReconciler.ReconcileSteps = []controllers.ReconcileStepFunc{
 		controllers.HandleFinalizers(c.finalizers),
 		controllers.RetrieveRevisionStates(revisionStatesGetter),
-		controllers.ResolveBundle(c.resolver),
+		controllers.ResolveBundle(c.resolver, c.mgr.GetClient()),
 		controllers.UnpackBundle(c.imagePuller, c.imageCache),
 		controllers.ApplyBundle(appl),
 	}

internal/operator-controller/applier/boxcutter.go

@@ -303,22 +303,35 @@ func (bc *Boxcutter) createOrUpdate(ctx context.Context, obj client.Object) erro
 }
 
 func (bc *Boxcutter) apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExtension, objectLabels, revisionAnnotations map[string]string) (bool, string, error) {
-	// Generate desired revision
-	desiredRevision, err := bc.RevisionGenerator.GenerateRevision(ctx, contentFS, ext, objectLabels, revisionAnnotations)
+	// List all existing revisions
+	existingRevisions, err := bc.getExistingRevisions(ctx, ext.GetName())
 	if err != nil {
 		return false, "", err
 	}
 
-	if err := controllerutil.SetControllerReference(ext, desiredRevision, bc.Scheme); err != nil {
-		return false, "", fmt.Errorf("set ownerref: %w", err)
+	// If contentFS is nil, we're maintaining the current state without catalog access.
+	// In this case, we should use the existing installed revision without generating a new one.
+	if contentFS == nil {
+		if len(existingRevisions) == 0 {
+			return false, "", fmt.Errorf("cannot maintain workload: no catalog content available and no previously installed revision found")
+		}
+		// Use the most recent revision and rely on its existing controller loop (don't create a new one).
+		// Returning true here signals that the rollout has succeeded using the current revision; the
+		// ClusterExtensionRevision controller will continue to reconcile and maintain the resources
+		// independently of this apply call.
+		return true, "", nil
 	}
 
-	// List all existing revisions
-	existingRevisions, err := bc.getExistingRevisions(ctx, ext.GetName())
+	// Generate desired revision
+	desiredRevision, err := bc.RevisionGenerator.GenerateRevision(ctx, contentFS, ext, objectLabels, revisionAnnotations)
 	if err != nil {
 		return false, "", err
 	}
 
+	if err := controllerutil.SetControllerReference(ext, desiredRevision, bc.Scheme); err != nil {
+		return false, "", fmt.Errorf("set ownerref: %w", err)
+	}
+
 	currentRevision := &ocv1.ClusterExtensionRevision{}
 	state := StateNeedsInstall
 	// check if we can update the current revision.

internal/operator-controller/applier/helm.go

@@ -103,6 +103,16 @@ func (h *Helm) runPreAuthorizationChecks(ctx context.Context, ext *ocv1.ClusterE
 }
 
 func (h *Helm) Apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExtension, objectLabels map[string]string, storageLabels map[string]string) (bool, string, error) {
+	// If contentFS is nil, we're maintaining the current state without catalog access.
+	// In this case, reconcile the existing Helm release if it exists.
+	if contentFS == nil {
+		ac, err := h.ActionClientGetter.ActionClientFor(ctx, ext)
+		if err != nil {
+			return false, "", err
+		}
+		return h.reconcileExistingRelease(ctx, ac, ext)
+	}
+
 	chrt, err := h.buildHelmChart(contentFS, ext)
 	if err != nil {
 		return false, "", err
@@ -197,6 +207,45 @@ func (h *Helm) Apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExte
 	return true, "", nil
 }
 
+// reconcileExistingRelease reconciles an existing Helm release without catalog access.
+// This is used when the catalog is unavailable but we need to maintain the current installation.
+// It reconciles the release and sets up watchers to ensure resources are maintained.
+func (h *Helm) reconcileExistingRelease(ctx context.Context, ac helmclient.ActionInterface, ext *ocv1.ClusterExtension) (bool, string, error) {
+	rel, err := ac.Get(ext.GetName())
+	if errors.Is(err, driver.ErrReleaseNotFound) {
+		return false, "", fmt.Errorf("cannot maintain workload: no catalog content available and no previously installed Helm release found")
+	}
+	if err != nil {
+		return false, "", fmt.Errorf("getting current release: %w", err)
+	}
+
+	// Reconcile the existing release to ensure resources are maintained
+	if err := ac.Reconcile(rel); err != nil {
+		// Reconcile failed - resources NOT maintained
+		// Return false (rollout failed) with error
+		return false, "", err
+	}
+
+	// At this point: Reconcile succeeded - resources ARE maintained
+	// The operations below are for setting up monitoring (watches).
+	// If they fail, the resources are still successfully reconciled and maintained,
+	// so we return true (rollout succeeded) even though monitoring setup failed.
+	relObjects, err := util.ManifestObjects(strings.NewReader(rel.Manifest), fmt.Sprintf("%s-release-manifest", rel.Name))
+	if err != nil {
+		return true, "", err
+	}
+	klog.FromContext(ctx).Info("watching managed objects")
+	cache, err := h.Manager.Get(ctx, ext)
+	if err != nil {
+		return true, "", err
+	}
+	if err := cache.Watch(ctx, h.Watcher, relObjects...); err != nil {
+		return true, "", err
+	}
+
+	return true, "", nil
+}
+
 func (h *Helm) buildHelmChart(bundleFS fs.FS, ext *ocv1.ClusterExtension) (*chart.Chart, error) {
 	if h.HelmChartProvider == nil {
 		return nil, errors.New("HelmChartProvider is nil")

internal/operator-controller/controllers/clusterextension_controller.go

@@ -168,6 +168,8 @@ func (r *ClusterExtensionReconciler) Reconcile(ctx context.Context, req ctrl.Req
 
 // ensureAllConditionsWithReason checks that all defined condition types exist in the given ClusterExtension,
 // and assigns a specified reason and custom message to any missing condition.
+//
+//nolint:unparam // reason parameter is designed to be flexible, even if current callers use the same value
 func ensureAllConditionsWithReason(ext *ocv1.ClusterExtension, reason v1alpha1.ConditionReason, message string) {
 	for _, condType := range conditionsets.ConditionTypes {
 		cond := apimeta.FindStatusCondition(ext.Status.Conditions, condType)

internal/operator-controller/controllers/clusterextension_controller_test.go

@@ -1611,3 +1611,189 @@ func TestGetInstalledBundleHistory(t *testing.T) {
 		}
 	}
 }
+
+// TestResolutionFallbackToInstalledBundle tests the catalog deletion resilience fallback logic
+func TestResolutionFallbackToInstalledBundle(t *testing.T) {
+	t.Run("falls back when catalog unavailable and no version change", func(t *testing.T) {
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			// Resolver fails (simulating catalog unavailable)
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				return nil, nil, nil, fmt.Errorf("catalog unavailable")
+			})
+			// Applier succeeds (resources maintained)
+			d.Applier = &MockApplier{
+				installCompleted: true,
+				installStatus:    "",
+				err:              nil,
+			}
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						Package:        "test-pkg",
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+						Image:          "test-image:1.0.0",
+					},
+				},
+			}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		// Create ClusterExtension with no version specified
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						// No version - should fall back
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// Reconcile should succeed (fallback to installed, then apply succeeds)
+		// Catalog watch will trigger reconciliation when catalog becomes available again
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Verify status shows successful reconciliation
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+
+		// Progressing should be Succeeded (apply completed successfully)
+		progCond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeProgressing)
+		require.NotNil(t, progCond)
+		require.Equal(t, metav1.ConditionTrue, progCond.Status)
+		require.Equal(t, ocv1.ReasonSucceeded, progCond.Reason)
+
+		// Installed should be True (maintaining current version)
+		instCond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeInstalled)
+		require.NotNil(t, instCond)
+		require.Equal(t, metav1.ConditionTrue, instCond.Status)
+		require.Equal(t, ocv1.ReasonSucceeded, instCond.Reason)
+	})
+
+	t.Run("fails when version upgrade requested without catalog", func(t *testing.T) {
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				return nil, nil, nil, fmt.Errorf("catalog unavailable")
+			})
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+					},
+				},
+			}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		// Create ClusterExtension requesting version upgrade
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						Version:     "1.0.1", // Requesting upgrade
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// Reconcile should fail (can't upgrade without catalog)
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.Error(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Verify status shows Retrying
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		cond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeProgressing)
+		require.NotNil(t, cond)
+		require.Equal(t, metav1.ConditionTrue, cond.Status)
+		require.Equal(t, ocv1.ReasonRetrying, cond.Reason)
+	})
+
+	t.Run("auto-updates when catalog becomes available after fallback", func(t *testing.T) {
+		resolveAttempt := 0
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			// First attempt: catalog unavailable, then becomes available
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				resolveAttempt++
+				if resolveAttempt == 1 {
+					// First reconcile: catalog unavailable
+					return nil, nil, nil, fmt.Errorf("catalog temporarily unavailable")
+				}
+				// Second reconcile (triggered by catalog watch): catalog available with new version
+				v := bundle.VersionRelease{Version: bsemver.MustParse("2.0.0")}
+				return &declcfg.Bundle{
+					Name:    "test.2.0.0",
+					Package: "test-pkg",
+					Image:   "test-image:2.0.0",
+				}, &v, nil, nil
+			})
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						Package:        "test-pkg",
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+						Image:          "test-image:1.0.0",
+					},
+				},
+			}
+			d.ImagePuller = &imageutil.MockPuller{ImageFS: fstest.MapFS{}}
+			d.Applier = &MockApplier{installCompleted: true}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						// No version - auto-update to latest
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// First reconcile: catalog unavailable, falls back to v1.0.0
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		require.Equal(t, "1.0.0", ext.Status.Install.Bundle.Version)
+
+		// Second reconcile: simulating catalog watch trigger, catalog now available with v2.0.0
+		res, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Should have upgraded to v2.0.0
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		require.Equal(t, "2.0.0", ext.Status.Install.Bundle.Version)
+
+		// Verify resolution was attempted twice (fallback, then success)
+		require.Equal(t, 2, resolveAttempt)
+	})
+}