e2e: convert all tests to per-scenario dynamic catalogs

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: joelanford

Makefile

@@ -320,7 +320,7 @@ test-e2e: GO_BUILD_EXTRA_FLAGS := -cover
 test-e2e: COVERAGE_NAME := e2e
 test-e2e: export MANIFEST := $(STANDARD_RELEASE_MANIFEST)
 test-e2e: export INSTALL_DEFAULT_CATALOGS := false
-test-e2e: run-internal image-registry prometheus e2e e2e-coverage kind-clean #HELP Run e2e test suite on local kind cluster
+test-e2e: run-internal prometheus e2e e2e-coverage kind-clean #HELP Run e2e test suite on local kind cluster
 
 .PHONY: test-experimental-e2e
 test-experimental-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_E2E_MANIFEST)
@@ -332,7 +332,7 @@ test-experimental-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST)
 test-experimental-e2e: export INSTALL_DEFAULT_CATALOGS := false
 test-experimental-e2e: PROMETHEUS_VALUES := helm/prom_experimental.yaml
 test-experimental-e2e: E2E_TIMEOUT := 15m
-test-experimental-e2e: run-internal image-registry prometheus e2e e2e-coverage kind-clean #HELP Run experimental e2e test suite on local kind cluster
+test-experimental-e2e: run-internal prometheus e2e e2e-coverage kind-clean #HELP Run experimental e2e test suite on local kind cluster
 
 .PHONY: prometheus
 prometheus: PROMETHEUS_NAMESPACE := olmv1-system

test/e2e/README.md

@@ -32,15 +32,13 @@ test/e2e/
 └── steps/                      # Step definitions and test utilities
     ├── steps.go                # Step definition implementations
     ├── hooks.go                # Test hooks and scenario context
-    └── testdata/               # Test data (RBAC templates, catalogs)
+    └── testdata/               # Test data (RBAC templates)
         ├── serviceaccount-template.yaml
         ├── olm-sa-helm-rbac-template.yaml
         ├── olm-sa-boxcutter-rbac-template.yaml
         ├── pvc-probe-sa-boxcutter-rbac-template.yaml
         ├── cluster-admin-rbac-template.yaml
-        ├── metrics-reader-rbac-template.yaml
-        ├── test-catalog-template.yaml
-        └── extra-catalog-template.yaml
+        └── metrics-reader-rbac-template.yaml
 ```
 
 ## Architecture
@@ -149,7 +147,7 @@ Manages test lifecycle and scenario-specific context.
 
 **Variable Substitution:**
 
-Replaces `${TEST_NAMESPACE}`, `${NAME}`, and `${CATALOG_IMG}` with scenario-specific values.
+Replaces `${TEST_NAMESPACE}`, `${NAME}`, `${SCENARIO_ID}`, `${PACKAGE:<name>}`, and `${CATALOG:<name>}` with scenario-specific values.
 
 ## Writing Tests
 
@@ -203,7 +201,9 @@ Use these variables in YAML templates:
 - `${NAME}`: Scenario-specific ClusterExtension name (e.g., `ce-123`)
 - `${COS_NAME}`: Scenario-specific ClusterObjectSet name (e.g., `cos-123`; for applying ClusterObjectSets directly)
 - `${TEST_NAMESPACE}`: Scenario-specific namespace (e.g., `ns-123`)
-- `${CATALOG_IMG}`: Catalog image reference (defaults to in-cluster registry, overridable via `CATALOG_IMG` env var)
+- `${SCENARIO_ID}`: Unique scenario identifier used for resource name isolation
+- `${PACKAGE:<name>}`: Parameterized package name (e.g., `${PACKAGE:test}` expands to `test-<scenario-id>`)
+- `${CATALOG:<name>}`: Catalog resource name (e.g., `${CATALOG:test}` expands to `test-catalog-<scenario-id>`)
 
 ### 5. Feature Tags
 
@@ -277,8 +277,8 @@ go test test/e2e/features_test.go --log.debug --k8s.cli=oc
 
 - `KUBECONFIG`: Path to kubeconfig file (defaults to `~/.kube/config`)
 - `E2E_SUMMARY_OUTPUT`: Path to write test summary (optional)
-- `CATALOG_IMG`: Override default catalog image reference (optional)
-- `LOCAL_REGISTRY_HOST`: Local registry host for catalog images
+- `LOCAL_REGISTRY_HOST`: Local registry host for pushing catalog images (default: `localhost:30000`)
+- `CLUSTER_REGISTRY_HOST`: In-cluster registry host for pulling catalog images
 
 ## Design Patterns
 
@@ -293,8 +293,8 @@ Each scenario runs in its own namespace with unique resource names, ensuring com
 
 The `ScenarioCleanup` hook ensures all resources are deleted after each scenario:
 
-- Kills background processes (e.g., kubectl port-forward)
-- Deletes ClusterExtensions
+- Deletes ClusterExtensions and ClusterObjectSets
+- Deletes ClusterCatalogs
 - Deletes namespaces
 - Deletes added resources
 

test/e2e/features/install.feature

@@ -2,7 +2,10 @@ Feature: Recover cluster extension from errors that might occur during its lifet
 
   Background:
     Given OLM is available
-    And ClusterCatalog "test" serves bundles
+    And an image registry is available
+    And a catalog "test" with packages:
+      | package | version | channel | replaces | contents                   |
+      | test    | 1.2.0   | beta    |          | CRD, Deployment, ConfigMap |
 
   Scenario: Restore removed resource
     Given ServiceAccount "olm-sa" with needed permissions is available in test namespace
@@ -19,15 +22,15 @@ Feature: Recover cluster extension from errors that might occur during its lifet
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
       """
     And ClusterExtension is available
-    And resource "configmap/test-configmap" exists
-    When resource "configmap/test-configmap" is removed
-    Then resource "configmap/test-configmap" is eventually restored
+    And resource "configmap/test-configmap-${SCENARIO_ID}" exists
+    When resource "configmap/test-configmap-${SCENARIO_ID}" is removed
+    Then resource "configmap/test-configmap-${SCENARIO_ID}" is eventually restored
 
   Scenario: Install ClusterExtension after target namespace becomes available
     Given ClusterExtension is applied
@@ -43,10 +46,10 @@ Feature: Recover cluster extension from errors that might occur during its lifet
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
       """
     And ClusterExtension reports Progressing as True with Reason Retrying
     When ServiceAccount "olm-sa" with needed permissions is available in test namespace
@@ -60,17 +63,17 @@ Feature: Recover cluster extension from errors that might occur during its lifet
       apiVersion: apps/v1
       kind: Deployment
       metadata:
-        name: test-operator
+        name: test-operator-${SCENARIO_ID}
         namespace: ${TEST_NAMESPACE}
       spec:
         replicas: 1
         selector:
           matchLabels:
-            app: test-operator
+            app: test-operator-${SCENARIO_ID}
         template:
           metadata:
             labels:
-              app: test-operator
+              app: test-operator-${SCENARIO_ID}
           spec:
             containers:
               - command:
@@ -103,14 +106,14 @@ Feature: Recover cluster extension from errors that might occur during its lifet
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
       """
     And ClusterExtension reports Progressing as True with Reason Retrying
     And ClusterExtension reports Installed as False
-    When resource "deployment/test-operator" is removed
+    When resource "deployment/test-operator-${SCENARIO_ID}" is removed
     Then ClusterExtension is available
     And ClusterExtension reports Progressing as True with Reason Succeeded
     And ClusterExtension reports Installed as True
@@ -132,26 +135,26 @@ Feature: Recover cluster extension from errors that might occur during its lifet
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
       """
     And ClusterExtension reports Progressing as True with Reason Retrying and Message includes:
       """
-      error for resolved bundle "test-operator.1.2.0" with version "1.2.0": creating new Revision: pre-authorization failed: service account requires the following permissions to manage cluster extension:
+      pre-authorization failed: service account requires the following permissions to manage cluster extension:
       """
     And ClusterExtension reports Progressing as True with Reason Retrying and Message includes:
       """
-      Namespace:"" APIGroups:[apiextensions.k8s.io] Resources:[customresourcedefinitions] ResourceNames:[olme2etests.olm.operatorframework.io] Verbs:[delete,get,patch,update]
+      Namespace:"" APIGroups:[apiextensions.k8s.io] Resources:[customresourcedefinitions] ResourceNames:[e2e-${SCENARIO_ID}tests.e2e-${SCENARIO_ID}.e2e.operatorframework.io] Verbs:[delete,get,patch,update]
       """
     When ServiceAccount "olm-sa" with needed permissions is available in test namespace
     Then ClusterExtension is available
     And ClusterExtension reports Progressing as True with Reason Succeeded
     And ClusterExtension reports Installed as True
 
   # CATALOG DELETION RESILIENCE SCENARIOS
-  
+
   Scenario: Auto-healing continues working after catalog deletion
     # This test proves that extensions continue to auto-heal (restore deleted resources) even when
     # their source catalog is unavailable. We verify this by:
@@ -177,19 +180,19 @@ Feature: Recover cluster extension from errors that might occur during its lifet
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
       """
     And ClusterExtension is rolled out
     And ClusterExtension is available
-    And resource "deployment/test-operator" is available
-    And resource "configmap/test-configmap" is available
-    When ClusterCatalog "test" is deleted
-    And resource "configmap/test-configmap" is removed
-    Then resource "configmap/test-configmap" is eventually restored
-    And resource "deployment/test-operator" is available
+    And resource "deployment/test-operator-${SCENARIO_ID}" is available
+    And resource "configmap/test-configmap-${SCENARIO_ID}" is available
+    When catalog "test" is deleted
+    And resource "configmap/test-configmap-${SCENARIO_ID}" is removed
+    Then resource "configmap/test-configmap-${SCENARIO_ID}" is eventually restored
+    And resource "deployment/test-operator-${SCENARIO_ID}" is available
 
   Scenario: Spec changes are allowed when catalog is unavailable
     # This test proves that users can modify extension configuration (non-version changes) even when
@@ -217,14 +220,14 @@ Feature: Recover cluster extension from errors that might occur during its lifet
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
       """
     And ClusterExtension is rolled out
     And ClusterExtension is available
-    And ClusterCatalog "test" is deleted
+    And catalog "test" is deleted
     When ClusterExtension is updated to add preflight config
       """
       apiVersion: olm.operatorframework.io/v1
@@ -242,10 +245,10 @@ Feature: Recover cluster extension from errors that might occur during its lifet
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
       """
     And ClusterExtension latest generation has been reconciled
     And ClusterExtension reports Progressing as True with Reason Succeeded

test/e2e/features/recover.feature

@@ -5,7 +5,10 @@ Feature: Report status of the managed ClusterExtension workload
 
   Background:
     Given OLM is available
-    And ClusterCatalog "test" serves bundles
+    And an image registry is available
+    And a catalog "test" with packages:
+      | package | version | channel | replaces | contents                   |
+      | test    | 1.0.0   | alpha   |          | CRD, Deployment, ConfigMap |
     And ServiceAccount "olm-sa" with needed permissions is available in test namespace
     And ClusterExtension is applied
       """
@@ -20,26 +23,26 @@ Feature: Report status of the managed ClusterExtension workload
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
             version: 1.0.0
       """
     And ClusterExtension is rolled out
     And ClusterExtension is available
 
   @BoxcutterRuntime
   Scenario: Report availability change when managed workload is not ready
-    When resource "deployment/test-operator" reports as not ready
+    When deployment "test-operator-${SCENARIO_ID}" reports as not ready
     Then ClusterExtension reports Available as False with Reason ProbeFailure
     And ClusterObjectSet "${NAME}-1" reports Available as False with Reason ProbeFailure
 
   @BoxcutterRuntime
   Scenario: Report availability change when managed workload restores its readiness
-    Given resource "deployment/test-operator" reports as not ready
+    Given deployment "test-operator-${SCENARIO_ID}" reports as not ready
     And ClusterExtension reports Available as False with Reason ProbeFailure
     And ClusterObjectSet "${NAME}-1" reports Available as False with Reason ProbeFailure
-    When resource "deployment/test-operator" reports as ready
+    When deployment "test-operator-${SCENARIO_ID}" reports as ready
     Then ClusterExtension is available
-    And ClusterObjectSet "${NAME}-1" reports Available as True with Reason ProbesSucceeded
+    And ClusterObjectSet "${NAME}-1" reports Available as True with Reason ProbesSucceeded

test/e2e/features/status.feature

@@ -5,7 +5,10 @@ Feature: Uninstall ClusterExtension
 
   Background:
     Given OLM is available
-    And ClusterCatalog "test" serves bundles
+    And an image registry is available
+    And a catalog "test" with packages:
+      | package | version | channel | replaces | contents                   |
+      | test    | 1.2.0   | beta    |          | CRD, Deployment, ConfigMap |
     And ServiceAccount "olm-sa" with needed permissions is available in test namespace
     And ClusterExtension is applied
       """
@@ -20,12 +23,12 @@ Feature: Uninstall ClusterExtension
         source:
           sourceType: Catalog
           catalog:
-            packageName: test
+            packageName: ${PACKAGE:test}
             selector:
               matchLabels:
-                "olm.operatorframework.io/metadata.name": test-catalog
+                "olm.operatorframework.io/metadata.name": ${CATALOG:test}
       """
-    And bundle "test-operator.1.2.0" is installed in version "1.2.0"
+    And bundle "${PACKAGE:test}.1.2.0" is installed in version "1.2.0"
     And ClusterExtension is rolled out
     And ClusterExtension resources are created and labeled