diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go
index ceb1450d1b..5ab1decac4 100644
--- a/pkg/manager/internal.go
+++ b/pkg/manager/internal.go
@@ -279,7 +279,7 @@ func (cm *controllerManager) GetAPIReader() client.Reader {
 func (cm *controllerManager) GetWebhookServer() webhook.Server {
 	cm.webhookServerOnce.Do(func() {
 		if cm.webhookServer == nil {
-			panic("webhook should not be nil")
+			return
 		}
 		if err := cm.Add(cm.webhookServer); err != nil {
 			panic(fmt.Sprintf("unable to add webhook server to the controller manager: %s", err))
diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go
index af532ea741..7bd4e7e129 100644
--- a/pkg/manager/manager.go
+++ b/pkg/manager/manager.go
@@ -443,6 +443,9 @@ func New(config *rest.Config, options Options) (Manager, error) {
 
 	errChan := make(chan error, 1)
 	runnables := newRunnables(options.BaseContext, errChan).withLogger(options.Logger)
+	if _, ok := options.WebhookServer.(*webhook.DisabledServer); ok {
+		options.WebhookServer = nil
+	}
 	return &controllerManager{
 		stopProcedureEngaged:          ptr.To(int64(0)),
 		cluster:                       cluster,
diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go
index 4bf553572d..32775e7c80 100644
--- a/pkg/manager/manager_test.go
+++ b/pkg/manager/manager_test.go
@@ -159,6 +159,16 @@ var _ = Describe("manger.Manager", func() {
 			_, isCustomWebhook := svr.(customWebhook)
 			Expect(isCustomWebhook).To(BeTrue())
 		})
+		It("should disable the webhook", func() {
+			By("setting the port to -1", func() {
+				m, err := New(cfg, Options{WebhookServer: webhook.NewServer(webhook.Options{Port: -1})})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(m).NotTo(BeNil())
+
+				svr := m.GetWebhookServer()
+				Expect(svr).To(BeNil())
+			})
+		})
 
 		Context("with leader election enabled", func() {
 			It("should only cancel the leader election after all runnables are done", func(specCtx SpecContext) {
diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go
index 079f0a55ff..86432dc83e 100644
--- a/pkg/webhook/server.go
+++ b/pkg/webhook/server.go
@@ -75,6 +75,8 @@ type Options struct {
 
 	// Port is the port number that the server will serve.
 	// It will be defaulted to 9443 if unspecified.
+	//
+	// To disable the webhook server set Port to -1.
 	Port int
 
 	// CertDir is the directory that contains the server key and certificate. Defaults to
@@ -105,6 +107,9 @@ type Options struct {
 
 // NewServer constructs a new webhook.Server from the provided options.
 func NewServer(o Options) Server {
+	if o.Port == -1 {
+		return &DisabledServer{}
+	}
 	return &DefaultServer{
 		Options: o,
 	}
@@ -300,3 +305,35 @@ func (s *DefaultServer) StartedChecker() healthz.Checker {
 func (s *DefaultServer) WebhookMux() *http.ServeMux {
 	return s.webhookMux
 }
+
+var _ Server = &DisabledServer{}
+
+// DisabledServer is a no-op implementation of Server
+// that can be used when you want to disable the webhook server.
+type DisabledServer struct{}
+
+// NeedLeaderElection returns false since the server is disabled and thus does not need leader election.
+func (*DisabledServer) NeedLeaderElection() bool {
+	return false
+}
+
+// Register returns immediately since the server is disabled and thus does not serve any webhooks.
+func (d *DisabledServer) Register(path string, hook http.Handler) {
+}
+
+// Start does nothing since the server is disabled and thus does not need to be started.
+func (d *DisabledServer) Start(ctx context.Context) error {
+	return nil
+}
+
+// StartedChecker returns nil since the server is disabled
+// and thus does not have a started state or healthz checker for whether the
+// server has been started.
+func (d *DisabledServer) StartedChecker() healthz.Checker {
+	return nil
+}
+
+// WebhookMux returns nil since the server is disabled and does not have a webhook mux.
+func (d *DisabledServer) WebhookMux() *http.ServeMux {
+	return nil
+}