🐛 Fix catalog deletion issues found in e2e tests

This fix resolves the issues discovered by the e2e tests, ensuring
Helm extensions keep running when catalogs are deleted.

Issues Fixed:

Issue 1: Extensions fail when catalog is deleted
  Problem: Resolution failure causes complete failure
  Fix: Fall back to installed bundle when catalog unavailable

Issue 2: Resources break when catalog is deleted
  Problem: No reconciliation without catalog content
  Fix: Helm reconcileExistingRelease() maintains resources

Issue 3: Config changes rejected without catalog
  Problem: Any spec change triggers resolution failure
  Fix: Allow non-version changes to proceed with installed bundle

Issue 4: Version upgrades fail silently
  Problem: Can't distinguish catalog deletion from other errors
  Fix: Check if catalogs exist, block upgrades explicitly

How the Fix Works:

When bundle resolution fails, the controller now:
1. Checks if a bundle is already installed
2. Checks if spec is requesting a version change
3. Checks if catalogs still exist in the cluster

Decision Logic:
- Installed + No version change + Catalog deleted → Use installed bundle
- Installed + Version change + Catalog deleted → Block and retry
- Installed + No version change + Catalog exists → Retry (transient error)
- No installed bundle → Fail (can't proceed)

For Helm Runtime:
- reconcileExistingRelease() uses Helm's built-in reconciliation
- Maintains resources via Server-Side Apply
- Sets up watchers to monitor and restore resources
- Returns success if reconcile succeeds (even if watch setup fails)

Controller watches ClusterCatalog resources, so reconciliation
automatically resumes when catalogs return.

Unit Tests Added:
- TestResolutionFallbackToInstalledBundle (3 scenarios)
- TestCheckCatalogsExist (5 scenarios)

All e2e tests now pass.

Author: camilamacedo86

cmd/operator-controller/main.go

@@ -626,7 +626,7 @@ func (c *boxcutterReconcilerConfigurator) Configure(ceReconciler *controllers.Cl
 		controllers.HandleFinalizers(c.finalizers),
 		controllers.MigrateStorage(storageMigrator),
 		controllers.RetrieveRevisionStates(revisionStatesGetter),
-		controllers.ResolveBundle(c.resolver),
+		controllers.ResolveBundle(c.resolver, c.mgr.GetClient()),
 		controllers.UnpackBundle(c.imagePuller, c.imageCache),
 		controllers.ApplyBundleWithBoxcutter(appl),
 	}
@@ -737,7 +737,7 @@ func (c *helmReconcilerConfigurator) Configure(ceReconciler *controllers.Cluster
 	ceReconciler.ReconcileSteps = []controllers.ReconcileStepFunc{
 		controllers.HandleFinalizers(c.finalizers),
 		controllers.RetrieveRevisionStates(revisionStatesGetter),
-		controllers.ResolveBundle(c.resolver),
+		controllers.ResolveBundle(c.resolver, c.mgr.GetClient()),
 		controllers.UnpackBundle(c.imagePuller, c.imageCache),
 		controllers.ApplyBundle(appl),
 	}

internal/operator-controller/applier/helm.go

@@ -103,6 +103,16 @@ func (h *Helm) runPreAuthorizationChecks(ctx context.Context, ext *ocv1.ClusterE
 }
 
 func (h *Helm) Apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExtension, objectLabels map[string]string, storageLabels map[string]string) (bool, string, error) {
+	// If contentFS is nil, we're maintaining the current state without catalog access.
+	// In this case, reconcile the existing Helm release if it exists.
+	if contentFS == nil {
+		ac, err := h.ActionClientGetter.ActionClientFor(ctx, ext)
+		if err != nil {
+			return false, "", err
+		}
+		return h.reconcileExistingRelease(ctx, ac, ext)
+	}
+
 	chrt, err := h.buildHelmChart(contentFS, ext)
 	if err != nil {
 		return false, "", err
@@ -197,6 +207,45 @@ func (h *Helm) Apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExte
 	return true, "", nil
 }
 
+// reconcileExistingRelease reconciles an existing Helm release without catalog access.
+// This is used when the catalog is unavailable but we need to maintain the current installation.
+// It reconciles the release and sets up watchers to ensure resources are maintained.
+func (h *Helm) reconcileExistingRelease(ctx context.Context, ac helmclient.ActionInterface, ext *ocv1.ClusterExtension) (bool, string, error) {
+	rel, err := ac.Get(ext.GetName())
+	if errors.Is(err, driver.ErrReleaseNotFound) {
+		return false, "", fmt.Errorf("cannot maintain workload: no catalog content available and no previously installed Helm release found")
+	}
+	if err != nil {
+		return false, "", fmt.Errorf("getting current release: %w", err)
+	}
+
+	// Reconcile the existing release to ensure resources are maintained
+	if err := ac.Reconcile(rel); err != nil {
+		// Reconcile failed - resources NOT maintained
+		// Return false (rollout failed) with error
+		return false, "", err
+	}
+
+	// At this point: Reconcile succeeded - resources ARE maintained
+	// The operations below are for setting up monitoring (watches).
+	// If they fail, the resources are still successfully reconciled and maintained,
+	// so we return true (rollout succeeded) even though monitoring setup failed.
+	relObjects, err := util.ManifestObjects(strings.NewReader(rel.Manifest), fmt.Sprintf("%s-release-manifest", rel.Name))
+	if err != nil {
+		return true, "", err
+	}
+	klog.FromContext(ctx).Info("watching managed objects")
+	cache, err := h.Manager.Get(ctx, ext)
+	if err != nil {
+		return true, "", err
+	}
+	if err := cache.Watch(ctx, h.Watcher, relObjects...); err != nil {
+		return true, "", err
+	}
+
+	return true, "", nil
+}
+
 func (h *Helm) buildHelmChart(bundleFS fs.FS, ext *ocv1.ClusterExtension) (*chart.Chart, error) {
 	if h.HelmChartProvider == nil {
 		return nil, errors.New("HelmChartProvider is nil")

internal/operator-controller/controllers/clusterextension_controller_test.go

@@ -1611,3 +1611,304 @@ func TestGetInstalledBundleHistory(t *testing.T) {
 		}
 	}
 }
+
+// TestResolutionFallbackToInstalledBundle tests the catalog deletion resilience fallback logic
+func TestResolutionFallbackToInstalledBundle(t *testing.T) {
+	t.Run("falls back when catalog unavailable and no version change", func(t *testing.T) {
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			// Resolver fails (simulating catalog unavailable)
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				return nil, nil, nil, fmt.Errorf("catalog unavailable")
+			})
+			// Applier succeeds (resources maintained)
+			d.Applier = &MockApplier{
+				installCompleted: true,
+				installStatus:    "",
+				err:              nil,
+			}
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						Package:        "test-pkg",
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+						Image:          "test-image:1.0.0",
+					},
+				},
+			}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		// Create ClusterExtension with no version specified
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						// No version - should fall back
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// Reconcile should succeed (fallback to installed, then apply succeeds)
+		// Catalog watch will trigger reconciliation when catalog becomes available again
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Verify status shows successful reconciliation
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+
+		// Progressing should be Succeeded (apply completed successfully)
+		progCond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeProgressing)
+		require.NotNil(t, progCond)
+		require.Equal(t, metav1.ConditionTrue, progCond.Status)
+		require.Equal(t, ocv1.ReasonSucceeded, progCond.Reason)
+
+		// Installed should be True (maintaining current version)
+		instCond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeInstalled)
+		require.NotNil(t, instCond)
+		require.Equal(t, metav1.ConditionTrue, instCond.Status)
+		require.Equal(t, ocv1.ReasonSucceeded, instCond.Reason)
+	})
+
+	t.Run("fails when version upgrade requested without catalog", func(t *testing.T) {
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				return nil, nil, nil, fmt.Errorf("catalog unavailable")
+			})
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+					},
+				},
+			}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		// Create ClusterExtension requesting version upgrade
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						Version:     "1.0.1", // Requesting upgrade
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// Reconcile should fail (can't upgrade without catalog)
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.Error(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Verify status shows Retrying
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		cond := apimeta.FindStatusCondition(ext.Status.Conditions, ocv1.TypeProgressing)
+		require.NotNil(t, cond)
+		require.Equal(t, metav1.ConditionTrue, cond.Status)
+		require.Equal(t, ocv1.ReasonRetrying, cond.Reason)
+	})
+
+	t.Run("auto-updates when catalog becomes available after fallback", func(t *testing.T) {
+		resolveAttempt := 0
+		cl, reconciler := newClientAndReconciler(t, func(d *deps) {
+			// First attempt: catalog unavailable, then becomes available
+			d.Resolver = resolve.Func(func(_ context.Context, _ *ocv1.ClusterExtension, _ *ocv1.BundleMetadata) (*declcfg.Bundle, *bundle.VersionRelease, *declcfg.Deprecation, error) {
+				resolveAttempt++
+				if resolveAttempt == 1 {
+					// First reconcile: catalog unavailable
+					return nil, nil, nil, fmt.Errorf("catalog temporarily unavailable")
+				}
+				// Second reconcile (triggered by catalog watch): catalog available with new version
+				v := bundle.VersionRelease{Version: bsemver.MustParse("2.0.0")}
+				return &declcfg.Bundle{
+					Name:    "test.2.0.0",
+					Package: "test-pkg",
+					Image:   "test-image:2.0.0",
+				}, &v, nil, nil
+			})
+			d.RevisionStatesGetter = &MockRevisionStatesGetter{
+				RevisionStates: &controllers.RevisionStates{
+					Installed: &controllers.RevisionMetadata{
+						Package:        "test-pkg",
+						BundleMetadata: ocv1.BundleMetadata{Name: "test.1.0.0", Version: "1.0.0"},
+						Image:          "test-image:1.0.0",
+					},
+				},
+			}
+			d.ImagePuller = &imageutil.MockPuller{ImageFS: fstest.MapFS{}}
+			d.Applier = &MockApplier{installCompleted: true}
+		})
+
+		ctx := context.Background()
+		extKey := types.NamespacedName{Name: fmt.Sprintf("test-%s", rand.String(8))}
+
+		ext := &ocv1.ClusterExtension{
+			ObjectMeta: metav1.ObjectMeta{Name: extKey.Name},
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						// No version - auto-update to latest
+					},
+				},
+				Namespace:      "default",
+				ServiceAccount: ocv1.ServiceAccountReference{Name: "default"},
+			},
+		}
+		require.NoError(t, cl.Create(ctx, ext))
+
+		// First reconcile: catalog unavailable, falls back to v1.0.0
+		res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		require.Equal(t, "1.0.0", ext.Status.Install.Bundle.Version)
+
+		// Second reconcile: simulating catalog watch trigger, catalog now available with v2.0.0
+		res, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: extKey})
+		require.NoError(t, err)
+		require.Equal(t, ctrl.Result{}, res)
+
+		// Should have upgraded to v2.0.0
+		require.NoError(t, cl.Get(ctx, extKey, ext))
+		require.Equal(t, "2.0.0", ext.Status.Install.Bundle.Version)
+
+		// Verify resolution was attempted twice (fallback, then success)
+		require.Equal(t, 2, resolveAttempt)
+	})
+}
+
+func TestCheckCatalogsExist(t *testing.T) {
+	t.Run("returns false when no catalogs exist", func(t *testing.T) {
+		cl := newClient(t)
+		ctx := context.Background()
+
+		ext := &ocv1.ClusterExtension{
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+					},
+				},
+			},
+		}
+
+		exists, err := controllers.CheckCatalogsExist(ctx, cl, ext)
+		require.NoError(t, err)
+		require.False(t, exists, "should return false when no catalogs exist")
+	})
+
+	t.Run("returns false when CRD doesn't exist", func(t *testing.T) {
+		cl := newClient(t)
+		ctx := context.Background()
+
+		ext := &ocv1.ClusterExtension{
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+					},
+				},
+			},
+		}
+
+		// ClusterCatalog CRD is not installed in test env
+		exists, err := controllers.CheckCatalogsExist(ctx, cl, ext)
+		require.NoError(t, err, "should not return error when CRD doesn't exist")
+		require.False(t, exists, "should treat missing CRD as no catalogs exist")
+	})
+
+	t.Run("returns false when no selector provided", func(t *testing.T) {
+		cl := newClient(t)
+		ctx := context.Background()
+
+		ext := &ocv1.ClusterExtension{
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						Selector:    nil, // No selector
+					},
+				},
+			},
+		}
+
+		exists, err := controllers.CheckCatalogsExist(ctx, cl, ext)
+		require.NoError(t, err)
+		require.False(t, exists, "should return false when no catalogs exist (no selector)")
+	})
+
+	t.Run("returns false when empty selector provided", func(t *testing.T) {
+		cl := newClient(t)
+		ctx := context.Background()
+
+		ext := &ocv1.ClusterExtension{
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						Selector:    &metav1.LabelSelector{}, // Empty selector (matches everything)
+					},
+				},
+			},
+		}
+
+		exists, err := controllers.CheckCatalogsExist(ctx, cl, ext)
+		require.NoError(t, err, "empty selector should not cause error")
+		require.False(t, exists, "should return false when no catalogs exist (empty selector)")
+	})
+
+	t.Run("returns error for invalid selector", func(t *testing.T) {
+		cl := newClient(t)
+		ctx := context.Background()
+
+		ext := &ocv1.ClusterExtension{
+			Spec: ocv1.ClusterExtensionSpec{
+				Source: ocv1.SourceConfig{
+					SourceType: "Catalog",
+					Catalog: &ocv1.CatalogFilter{
+						PackageName: "test-pkg",
+						Selector: &metav1.LabelSelector{
+							MatchExpressions: []metav1.LabelSelectorRequirement{
+								{
+									Key:      "invalid",
+									Operator: "InvalidOperator", // Invalid operator
+									Values:   []string{"value"},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		exists, err := controllers.CheckCatalogsExist(ctx, cl, ext)
+		require.Error(t, err, "should return error for invalid selector")
+		require.Contains(t, err.Error(), "invalid catalog selector")
+		require.False(t, exists)
+	})
+}