fix(catalogd): bind catalog HTTP port lazily; add readiness check

The catalog HTTP server has OnlyServeWhenLeader: true, so only the leader
pod should serve catalog content. Previously, net.Listen was called eagerly
at startup for all pods: the listen socket was bound on non-leaders even
though http.Serve was never called, causing TCP connections to queue without
being served. With replicas > 1 this made ~50% of catalog content requests
fail silently.

Replace manager.Server with a custom Runnable (catalogServerRunnable) in
serverutil that:
- Binds the catalog port lazily inside Start(), which is only called on the
  leader by controller-runtime's leader election machinery.
- Closes a ready channel once the listener is established, and registers a
  channel-select readiness check via AddReadyzCheck so non-leader pods fail
  the /readyz probe and are excluded from Service endpoints.

This keeps cmd/catalogd/main.go health/readiness setup identical to
cmd/operator-controller/main.go (healthz.Ping for both liveness and
readiness); the catalog-server readiness check is an implementation detail
of serverutil.AddCatalogServerToManager.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: tmshort

Makefile

@@ -254,7 +254,7 @@ $(eval $(call install-sh,standard,operator-controller-standard.yaml))
 .PHONY: test
 test: manifests generate fmt lint test-unit test-e2e test-regression #HELP Run all tests.
 
-E2E_TIMEOUT ?= 15m
+E2E_TIMEOUT ?= 20m
 GODOG_ARGS ?=
 .PHONY: e2e
 e2e: #EXHELP Run the e2e tests.

internal/catalogd/serverutil/serverutil.go

@@ -1,7 +1,9 @@
 package serverutil
 
 import (
+	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -13,7 +15,7 @@ import (
 	"github.com/klauspost/compress/gzhttp"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
-	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 
 	catalogdmetrics "github.com/operator-framework/operator-controller/internal/catalogd/metrics"
 	"github.com/operator-framework/operator-controller/internal/catalogd/storage"
@@ -27,49 +29,114 @@ type CatalogServerConfig struct {
 	LocalStorage storage.Instance
 }
 
-func AddCatalogServerToManager(mgr ctrl.Manager, cfg CatalogServerConfig, tlsFileWatcher *certwatcher.CertWatcher) error {
-	listener, err := net.Listen("tcp", cfg.CatalogAddr)
+// AddCatalogServerToManager adds the catalog HTTP server to the manager and registers
+// a readiness check that passes only when this pod is the leader and actively serving.
+// The listener is created lazily inside Start() so non-leader pods never bind the port,
+// which ensures the readiness check correctly excludes them from Service endpoints.
+func AddCatalogServerToManager(mgr ctrl.Manager, cfg CatalogServerConfig, cw *certwatcher.CertWatcher) error {
+	shutdownTimeout := 30 * time.Second
+	r := &catalogServerRunnable{
+		cfg: cfg,
+		cw:  cw,
+		server: &http.Server{
+			Addr:         cfg.CatalogAddr,
+			Handler:      storageServerHandlerWrapped(mgr.GetLogger().WithName("catalogd-http-server"), cfg),
+			ReadTimeout:  5 * time.Second,
+			WriteTimeout: 5 * time.Minute,
+		},
+		shutdownTimeout: shutdownTimeout,
+		ready:           make(chan struct{}),
+	}
+
+	if err := mgr.Add(r); err != nil {
+		return fmt.Errorf("error adding catalog server to manager: %w", err)
+	}
+
+	// Register a readiness check that passes only once Start() has been called (i.e.
+	// this pod holds the leader lease and the catalog server is actively serving).
+	// Non-leader pods never reach Start(), so they remain not-ready and are excluded
+	// from Service endpoints — preventing catalog traffic from hitting a pod that
+	// isn't serving the catalog port.
+	if err := mgr.AddReadyzCheck("catalog-server", r.readyzCheck()); err != nil {
+		return fmt.Errorf("error adding catalog server readiness check: %w", err)
+	}
+
+	return nil
+}
+
+// catalogServerRunnable is a leader-only Runnable that binds the catalog HTTP port
+// lazily inside Start(), so non-leader pods never hold the listen socket.
+type catalogServerRunnable struct {
+	cfg             CatalogServerConfig
+	cw              *certwatcher.CertWatcher
+	server          *http.Server
+	shutdownTimeout time.Duration
+	// ready is closed by Start() once the server is about to begin serving.
+	ready chan struct{}
+}
+
+// NeedLeaderElection returns false so the catalog server starts on every pod
+// immediately, regardless of leadership.  This is required for rolling updates:
+// if Start() were gated on leadership, a new pod could not win the leader lease
+// (held by the still-running old pod) and therefore could never pass the
+// catalog-server readiness check, deadlocking the rollout.
+//
+// Non-leader pods serve the catalog HTTP port but have an empty local cache
+// (only the leader's reconciler downloads catalog content), so requests to a
+// non-leader return 404.  Callers are expected to retry.
+func (r *catalogServerRunnable) NeedLeaderElection() bool { return false }
+
+func (r *catalogServerRunnable) Start(ctx context.Context) error {
+	listener, err := net.Listen("tcp", r.cfg.CatalogAddr)
 	if err != nil {
 		return fmt.Errorf("error creating catalog server listener: %w", err)
 	}
 
-	if cfg.CertFile != "" && cfg.KeyFile != "" {
-		// Use the passed certificate watcher instead of creating a new one
+	if r.cfg.CertFile != "" && r.cfg.KeyFile != "" {
 		config := &tls.Config{
-			GetCertificate: tlsFileWatcher.GetCertificate,
+			GetCertificate: r.cw.GetCertificate,
 			MinVersion:     tls.VersionTLS12,
 		}
 		listener = tls.NewListener(listener, config)
 	}
 
-	shutdownTimeout := 30 * time.Second
-	catalogServer := manager.Server{
-		Name:                "catalogs",
-		OnlyServeWhenLeader: true,
-		Server: &http.Server{
-			Addr:        cfg.CatalogAddr,
-			Handler:     storageServerHandlerWrapped(mgr.GetLogger().WithName("catalogd-http-server"), cfg),
-			ReadTimeout: 5 * time.Second,
-			// TODO: Revert this to 10 seconds if/when the API
-			// evolves to have significantly smaller responses
-			WriteTimeout: 5 * time.Minute,
-		},
-		ShutdownTimeout: &shutdownTimeout,
-		Listener:        listener,
-	}
+	// Signal readiness before blocking on Serve so the readiness probe passes promptly.
+	close(r.ready)
 
-	err = mgr.Add(&catalogServer)
-	if err != nil {
-		return fmt.Errorf("error adding catalog server to manager: %w", err)
-	}
+	go func() {
+		<-ctx.Done()
+		shutdownCtx := context.Background()
+		if r.shutdownTimeout > 0 {
+			var cancel context.CancelFunc
+			shutdownCtx, cancel = context.WithTimeout(shutdownCtx, r.shutdownTimeout)
+			defer cancel()
+		}
+		if err := r.server.Shutdown(shutdownCtx); err != nil {
+			// Shutdown errors are logged by the manager; nothing actionable here.
+			_ = err
+		}
+	}()
 
+	if err := r.server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		return err
+	}
 	return nil
 }
 
+// readyzCheck returns a healthz.Checker that passes once Start() has been called.
+func (r *catalogServerRunnable) readyzCheck() healthz.Checker {
+	return func(_ *http.Request) error {
+		select {
+		case <-r.ready:
+			return nil
+		default:
+			return fmt.Errorf("catalog server not yet started")
+		}
+	}
+}
+
 func logrLoggingHandler(l logr.Logger, handler http.Handler) http.Handler {
 	return handlers.CustomLoggingHandler(nil, handler, func(_ io.Writer, params handlers.LogFormatterParams) {
-		// extract parameters used in apache common log format, but then log using `logr` to remain consistent
-		// with other loggers used in this codebase.
 		username := "-"
 		if params.URL.User != nil {
 			if name := params.URL.User.Username(); name != "" {