Add auto-cleanup and improved error messaging for bundle unpack jobs

This change improves the bundle unpacking experience by adding automatic
cleanup of failed/completed jobs and providing better troubleshooting
guidance to users.

Changes:
- Set TTLSecondsAfterFinished to 300 seconds (5 minutes) for bundle
  unpack jobs to enable automatic retry when the root cause is fixed
- Enhanced error messages with detailed troubleshooting steps including:
  - Information about automatic retry in ~5 minutes
  - Manual job deletion commands for immediate retry
  - Common failure causes (ICSP, auth, image issues)
  - Specific guidance for ImageContentSourcePolicy configuration

The 5-minute TTL provides a balance between fast recovery and preventing
job churn on permanent failures.

Co-Authored-By: Rohit Patil <ropatil@redhat.com>

Author: Rohit Patil

pkg/controller/bundle/bundle_unpacker.go

@@ -102,7 +102,10 @@ func (c *ConfigMapUnpacker) job(cmRef *corev1.ObjectReference, bundlePath string
 			},
 		},
 		Spec: batchv1.JobSpec{
-			//ttlSecondsAfterFinished: 0 // can use in the future to not have to clean up job
+			// Auto-cleanup failed/completed jobs after 5 minutes to allow automatic retry when root cause is fixed
+			// 5 minutes provides balance between fast recovery and preventing job churn on permanent failures
+			// See: https://kubernetes.io/docs/concepts/workloads/controllers/job/#ttl-mechanism-for-finished-jobs
+			TTLSecondsAfterFinished: ptr.To[int32](300),
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: cmRef.Name,

pkg/controller/bundle/bundle_unpacker_test.go

@@ -50,6 +50,7 @@ func TestConfigMapUnpacker(t *testing.T) {
 		return start
 	}
 	backoffLimit := int32(3)
+	ttlSecondsAfterFinished := int32(300) // 5 minutes TTL for automatic cleanup
 	// Used to set the default value for job.spec.ActiveDeadlineSeconds
 	// that would normally be passed from the cmdline flag
 	defaultUnpackDuration := 10 * time.Minute
@@ -263,8 +264,9 @@ func TestConfigMapUnpacker(t *testing.T) {
 						},
 						Spec: batchv1.JobSpec{
 							// The expected job's timeout should be set to the custom annotation timeout
-							ActiveDeadlineSeconds: &customAnnotationTimeoutSeconds,
-							BackoffLimit:          &backoffLimit,
+							ActiveDeadlineSeconds:   &customAnnotationTimeoutSeconds,
+							BackoffLimit:            &backoffLimit,
+							TTLSecondsAfterFinished: &ttlSecondsAfterFinished,
 							Template: corev1.PodTemplateSpec{
 								ObjectMeta: metav1.ObjectMeta{
 									Name: pathHash,
@@ -484,8 +486,9 @@ func TestConfigMapUnpacker(t *testing.T) {
 							},
 						},
 						Spec: batchv1.JobSpec{
-							ActiveDeadlineSeconds: &defaultUnpackTimeoutSeconds,
-							BackoffLimit:          &backoffLimit,
+							ActiveDeadlineSeconds:   &defaultUnpackTimeoutSeconds,
+							BackoffLimit:            &backoffLimit,
+							TTLSecondsAfterFinished: &ttlSecondsAfterFinished,
 							Template: corev1.PodTemplateSpec{
 								ObjectMeta: metav1.ObjectMeta{
 									Name: digestHash,
@@ -744,8 +747,9 @@ func TestConfigMapUnpacker(t *testing.T) {
 							},
 						},
 						Spec: batchv1.JobSpec{
-							ActiveDeadlineSeconds: &defaultUnpackTimeoutSeconds,
-							BackoffLimit:          &backoffLimit,
+							ActiveDeadlineSeconds:   &defaultUnpackTimeoutSeconds,
+							BackoffLimit:            &backoffLimit,
+							TTLSecondsAfterFinished: &ttlSecondsAfterFinished,
 							Template: corev1.PodTemplateSpec{
 								ObjectMeta: metav1.ObjectMeta{
 									Name: digestHash,
@@ -999,8 +1003,9 @@ func TestConfigMapUnpacker(t *testing.T) {
 							},
 						},
 						Spec: batchv1.JobSpec{
-							ActiveDeadlineSeconds: &defaultUnpackTimeoutSeconds,
-							BackoffLimit:          &backoffLimit,
+							ActiveDeadlineSeconds:   &defaultUnpackTimeoutSeconds,
+							BackoffLimit:            &backoffLimit,
+							TTLSecondsAfterFinished: &ttlSecondsAfterFinished,
 							Template: corev1.PodTemplateSpec{
 								ObjectMeta: metav1.ObjectMeta{
 									Name: pathHash,
@@ -1224,8 +1229,9 @@ func TestConfigMapUnpacker(t *testing.T) {
 							},
 						},
 						Spec: batchv1.JobSpec{
-							ActiveDeadlineSeconds: &defaultUnpackTimeoutSeconds,
-							BackoffLimit:          &backoffLimit,
+							ActiveDeadlineSeconds:   &defaultUnpackTimeoutSeconds,
+							BackoffLimit:            &backoffLimit,
+							TTLSecondsAfterFinished: &ttlSecondsAfterFinished,
 							Template: corev1.PodTemplateSpec{
 								ObjectMeta: metav1.ObjectMeta{
 									Name: pathHash,
@@ -1462,8 +1468,9 @@ func TestConfigMapUnpacker(t *testing.T) {
 							},
 						},
 						Spec: batchv1.JobSpec{
-							ActiveDeadlineSeconds: &defaultUnpackTimeoutSeconds,
-							BackoffLimit:          &backoffLimit,
+							ActiveDeadlineSeconds:   &defaultUnpackTimeoutSeconds,
+							BackoffLimit:            &backoffLimit,
+							TTLSecondsAfterFinished: &ttlSecondsAfterFinished,
 							Template: corev1.PodTemplateSpec{
 								ObjectMeta: metav1.ObjectMeta{
 									Name: pathHash,
@@ -2075,3 +2082,43 @@ func TestSortUnpackJobs(t *testing.T) {
 		assert.ElementsMatch(t, tc.expectedToDelete, toDelete)
 	}
 }
+
+// TestBundleUnpackJobHasTTL verifies that bundle unpack jobs have TTL set for automatic cleanup
+func TestBundleUnpackJobHasTTL(t *testing.T) {
+	cmRef := &corev1.ObjectReference{
+		APIVersion: "v1",
+		Kind:       "ConfigMap",
+		Name:       "test-bundle",
+		Namespace:  "test-namespace",
+		UID:        "test-uid",
+	}
+	bundlePath := "quay.io/test/bundle:v1.0.0"
+	secrets := []corev1.LocalObjectReference{{Name: "test-secret"}}
+	timeout := 10 * time.Minute
+
+	unpacker := &ConfigMapUnpacker{
+		opmImage:      "test-opm:latest",
+		utilImage:     "test-util:latest",
+		unpackTimeout: timeout,
+		runAsUser:     1001,
+	}
+
+	job := unpacker.job(cmRef, bundlePath, secrets, timeout)
+
+	// Verify TTL is set to 5 minutes (300 seconds)
+	require.NotNil(t, job.Spec.TTLSecondsAfterFinished, "TTLSecondsAfterFinished should be set")
+	assert.Equal(t, int32(300), *job.Spec.TTLSecondsAfterFinished, "TTL should be 300 seconds (5 minutes) for faster recovery")
+	
+	// Verify other important job specs are still set correctly
+	assert.NotNil(t, job.Spec.BackoffLimit, "BackoffLimit should be set")
+	assert.Equal(t, int32(3), *job.Spec.BackoffLimit)
+	
+	assert.NotNil(t, job.Spec.ActiveDeadlineSeconds, "ActiveDeadlineSeconds should be set")
+	assert.Equal(t, int64(timeout.Seconds()), *job.Spec.ActiveDeadlineSeconds)
+	
+	// Verify job metadata
+	assert.Equal(t, cmRef.Name, job.Name)
+	assert.Equal(t, cmRef.Namespace, job.Namespace)
+	assert.Contains(t, job.Labels, install.OLMManagedLabelKey)
+	assert.Contains(t, job.Labels, BundleUnpackRefLabel)
+}

pkg/controller/operators/catalog/operator.go

@@ -1398,14 +1398,32 @@ func (o *Operator) syncResolvingNamespace(obj interface{}) error {
 		// with a condition indicating the failure.
 		isFailed, cond := hasBundleLookupFailureCondition(bundleLookups)
 		if isFailed {
-			err := fmt.Errorf("bundle unpacking failed. Reason: %v, and Message: %v", cond.Reason, cond.Message)
-			logger.Infof("%v", err)
+			// Create detailed error message with troubleshooting guidance
+			detailedMessage := fmt.Sprintf(`Bundle unpacking failed. Reason: %v, Message: %v
+
+The failed bundle unpack job will be automatically cleaned up and retried in approximately 5 minutes.
+
+To retry immediately, delete the failed job:
+  kubectl get jobs -n openshift-marketplace -l operatorframework.io/bundle-unpack-ref
+  kubectl delete job <job-name> -n openshift-marketplace
+
+Common causes:
+  • Missing ImageContentSourcePolicy for File-Based Catalogs (required for ppc64le, s390x, arm64)
+  • Image pull authentication failures
+  • Non-existent bundle image or incorrect image reference
+  • Network connectivity issues to image registry
+
+For ImageContentSourcePolicy issues:
+  1. Apply the correct ICSP configuration
+  2. Wait ~5 minutes for automatic retry OR delete the failed job manually`, cond.Reason, cond.Message)
+
+			logger.Infof("bundle unpacking failed. Reason: %v, Message: %v", cond.Reason, cond.Message)
 
 			_, updateErr := o.updateSubscriptionStatuses(
 				o.setSubsCond(subs, v1alpha1.SubscriptionCondition{
 					Type:    v1alpha1.SubscriptionBundleUnpackFailed,
 					Reason:  "BundleUnpackFailed",
-					Message: err.Error(),
+					Message: detailedMessage,
 					Status:  corev1.ConditionTrue,
 				}))
 			if updateErr != nil {

pkg/controller/operators/catalog/subscriptions_test.go

@@ -563,7 +563,7 @@ func TestSyncSubscriptions(t *testing.T) {
 							{
 								Type:    v1alpha1.SubscriptionBundleUnpackFailed,
 								Reason:  "BundleUnpackFailed",
-								Message: "bundle unpacking failed. Reason: JobFailed, and Message: unpack job failed",
+								Message: "Bundle unpacking failed. Reason: JobFailed, Message: unpack job failed\n\nThe failed bundle unpack job will be automatically cleaned up and retried in approximately 5 minutes.\n\nTo retry immediately, delete the failed job:\n  kubectl get jobs -n openshift-marketplace -l operatorframework.io/bundle-unpack-ref\n  kubectl delete job <job-name> -n openshift-marketplace\n\nCommon causes:\n  • Missing ImageContentSourcePolicy for File-Based Catalogs (required for ppc64le, s390x, arm64)\n  • Image pull authentication failures\n  • Non-existent bundle image or incorrect image reference\n  • Network connectivity issues to image registry\n\nFor ImageContentSourcePolicy issues:\n  1. Apply the correct ICSP configuration\n  2. Wait ~5 minutes for automatic retry OR delete the failed job manually",
 								Status:  corev1.ConditionTrue,
 							},
 						},