fix: align cert-manager versions, pre-import images, and auto-recreate stale k3d clusters

Root cause: CI agents persist k3d clusters across builds. When the
cert-manager version in the startup manifest changes, loadCluster
patches the HelmChart which triggers an in-place upgrade. This
disrupts the running webhook during transition, causing "cert-manager
webhook not ready" timeouts.

Fixes:
1. Align cert-manager to v1.17.2 (pre-pulled by CI) in both
   pkg/k3d/cert-manager.yaml and pkg/testutil/testutil.go.

2. Pre-import cert-manager images into k3d containerd in waitForJobs()
   so the helm controller doesn't need to pull from the internet.

3. Auto-recreate stale clusters: if loadCluster fails (e.g. webhook
   never becomes ready after manifest upgrade), GetOrCreate deletes
   the unhealthy cluster and creates a fresh one. This handles the
   case where a CI agent has a k3d cluster from a previous build with
   an incompatible cert-manager version.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: david-yu

pkg/k3d/cert-manager.yaml

@@ -9,7 +9,7 @@ spec:
   chart: cert-manager
   createNamespace: true
   targetNamespace: cert-manager
-  version: "v1.16.1"
+  version: "v1.17.2"
   valuesContent: |-
     crds:
       enabled: true

pkg/k3d/k3d.go

@@ -203,7 +203,23 @@ func GetOrCreate(name string, opts ...ClusterOpt) (*Cluster, error) {
 	cluster, err := NewCluster(name, opts...)
 	if err != nil {
 		if errors.Is(err, ErrExists) {
-			return loadCluster(name, config)
+			c, loadErr := loadCluster(name, config)
+			if loadErr != nil {
+				// Cluster exists but is unhealthy (e.g. stale cert-manager
+				// from a previous build). Delete and recreate.
+				fmt.Printf("WARNING: existing k3d cluster %q is unhealthy (%v), deleting and recreating\n", name, loadErr)
+				deleteCluster(name)
+				clearImageMarkers(name)
+				c2, err2 := NewCluster(name, opts...)
+				if err2 != nil {
+					return nil, errors.Wrapf(err2, "recreating cluster %q after unhealthy load", name)
+				}
+				if err2 := c2.importImages("localhost/redpanda-operator:dev"); err2 != nil {
+					return nil, err2
+				}
+				return c2, nil
+			}
+			return c, nil
 		}
 		return nil, err
 	}
@@ -215,6 +231,15 @@ func GetOrCreate(name string, opts ...ClusterOpt) (*Cluster, error) {
 	return cluster, nil
 }
 
+// deleteCluster removes a k3d cluster by name. Best-effort — errors are logged
+// but not returned, since this is used as cleanup before recreation.
+func deleteCluster(name string) {
+	out, err := exec.Command("k3d", "cluster", "delete", name).CombinedOutput()
+	if err != nil {
+		fmt.Printf("WARNING: failed to delete k3d cluster %q: %v: %s\n", name, err, out)
+	}
+}
+
 // imageMarkerPath returns a file path used to track whether an image has
 // already been imported into a given k3d cluster.
 func imageMarkerPath(clusterName, image string) string {
@@ -509,6 +534,17 @@ func (c *Cluster) waitForJobs(ctx context.Context) error {
 	}
 
 	if !c.skipManifests {
+		// Pre-import cert-manager images into the k3d cluster so the k3s
+		// helm controller doesn't need to pull them from the internet.
+		// The CI `test:pull-images` task downloads these into the host
+		// Docker daemon, but k3d's containerd is separate — images must
+		// be explicitly imported via `k3d image import`.
+		if err := c.importImages(certManagerImages()...); err != nil {
+			// Non-fatal: if the images aren't in the host Docker daemon
+			// (e.g. local dev), k3s will pull them from the registry.
+			fmt.Printf("WARNING: failed to import cert-manager images (will rely on registry pull): %v\n", err)
+		}
+
 		// NB: Originally this functionality was achieved via the --volume flag to
 		// k3d but CI runs via a docker in docker setup which makes it unreasonable
 		// to use --volume.
@@ -554,6 +590,19 @@ func (c *Cluster) waitForJobs(ctx context.Context) error {
 	return nil
 }
 
+// certManagerImages returns the container images needed by the cert-manager
+// version deployed via the startup manifests. These are imported into the k3d
+// cluster so the k3s helm controller doesn't need to pull from the internet.
+func certManagerImages() []string {
+	v := testutil.CertManagerVersion
+	return []string{
+		"quay.io/jetstack/cert-manager-controller:" + v,
+		"quay.io/jetstack/cert-manager-webhook:" + v,
+		"quay.io/jetstack/cert-manager-cainjector:" + v,
+		"quay.io/jetstack/cert-manager-startupapicheck:" + v,
+	}
+}
+
 // startupManifests parses the embedded FS of Kubernetes manifests as a slice
 // of [client.Object]s.
 var startupManifests = sync.OnceValue(func() []client.Object {

pkg/testutil/testutil.go

@@ -28,7 +28,7 @@ var (
 )
 
 const (
-	CertManagerVersion = "v1.17.1"
+	CertManagerVersion = "v1.17.2"
 	VClusterVersion    = "v0.31.2"
 )