Merge pull request #421 from pdettori/fix/webhook-readiness-420

Fix: add TLS connectivity probe to webhook readiness checks

Author: pdettori

kagenti-operator/cmd/main.go

@@ -688,6 +688,10 @@ func main() {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
+	if err := mgr.AddReadyzCheck("webhook", webhookServer.StartedChecker()); err != nil {
+		setupLog.Error(err, "unable to set up webhook ready check")
+		os.Exit(1)
+	}
 
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctx); err != nil {

kagenti-operator/test/e2e/e2e_test.go

@@ -344,6 +344,11 @@ var _ = Describe("AuthBridge Injection E2E", Ordered, func() {
 			g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated")
 		}, 2*time.Minute, 2*time.Second).Should(Succeed())
 
+		By("waiting for controller pod to be fully Ready")
+		Eventually(func() error {
+			return utils.ProbeWebhookReady(controllerNamespace)
+		}, 2*time.Minute, 2*time.Second).Should(Succeed())
+
 		By("creating auth bridge test namespace")
 		cmd := exec.Command("kubectl", "create", "ns", authBridgeTestNamespace)
 		_, err := utils.Run(cmd)
@@ -696,6 +701,11 @@ var _ = Describe("AgentCard E2E", Ordered, func() {
 			g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated")
 		}, 2*time.Minute, 2*time.Second).Should(Succeed())
 
+		By("waiting for controller pod to be fully Ready")
+		Eventually(func() error {
+			return utils.ProbeWebhookReady(controllerNamespace)
+		}, 2*time.Minute, 2*time.Second).Should(Succeed())
+
 		By("creating test namespace with labels")
 		cmd := exec.Command("kubectl", "create", "ns", testNamespace)
 		_, err := utils.Run(cmd)
@@ -1039,6 +1049,11 @@ rules:
 			g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated")
 		}, 2*time.Minute, 2*time.Second).Should(Succeed())
 
+		By("waiting for controller pod to be fully Ready")
+		Eventually(func() error {
+			return utils.ProbeWebhookReady(controllerNamespace)
+		}, 2*time.Minute, 2*time.Second).Should(Succeed())
+
 		By("creating test namespace")
 		cmd := exec.Command("kubectl", "create", "ns", agentRuntimeTestNamespace)
 		_, err := utils.Run(cmd)
@@ -1513,6 +1528,11 @@ rules:
 			g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated")
 		}, 2*time.Minute, 2*time.Second).Should(Succeed())
 
+		By("waiting for controller pod to be fully Ready")
+		Eventually(func() error {
+			return utils.ProbeWebhookReady(controllerNamespace)
+		}, 2*time.Minute, 2*time.Second).Should(Succeed())
+
 		By("setting KAGENTI_SPIRE_TRUST_DOMAIN env var")
 		envCmd := exec.Command("kubectl", "set", "env", "deployment/"+controllerDeployment,
 			"-n", controllerNamespace, "KAGENTI_SPIRE_TRUST_DOMAIN=example.org")
@@ -1974,6 +1994,11 @@ rules:
 			g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated")
 		}, 2*time.Minute, 2*time.Second).Should(Succeed())
 
+		By("waiting for controller pod to be fully Ready")
+		Eventually(func() error {
+			return utils.ProbeWebhookReady(controllerNamespace)
+		}, 2*time.Minute, 2*time.Second).Should(Succeed())
+
 		By("creating skill discovery test namespace")
 		cmd := exec.Command("kubectl", "create", "ns", skillDiscoveryTestNamespace)
 		_, err := utils.Run(cmd)
@@ -2134,6 +2159,11 @@ rules:
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated")
 			}, 2*time.Minute, 2*time.Second).Should(Succeed())
+
+			By("waiting for controller pod to be fully Ready after restart")
+			Eventually(func() error {
+				return utils.ProbeWebhookReady(controllerNamespace)
+			}, 2*time.Minute, 2*time.Second).Should(Succeed())
 		})
 
 		It("should populate linkedSkills from annotation", func() {
@@ -2438,6 +2468,11 @@ var _ = Describe("Istio Mesh Enrollment E2E", Ordered, func() {
 			g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated")
 		}, 2*time.Minute, 2*time.Second).Should(Succeed())
 
+		By("waiting for controller pod to be fully Ready")
+		Eventually(func() error {
+			return utils.ProbeWebhookReady(controllerNamespace)
+		}, 2*time.Minute, 2*time.Second).Should(Succeed())
+
 		By("creating test namespace")
 		cmd := exec.Command("kubectl", "create", "ns", istioMeshTestNamespace)
 		_, err := utils.Run(cmd)

kagenti-operator/test/utils/utils.go

@@ -751,3 +751,19 @@ func UncommentCode(filename, target, prefix string) error {
 	// nolint:gosec
 	return os.WriteFile(filename, out.Bytes(), 0644)
 }
+
+// ProbeWebhookReady checks that the controller pod is fully Ready.
+// The readyz endpoint gates on webhookServer.StartedChecker(), so Ready
+// means the webhook TLS server is accepting connections on :9443.
+func ProbeWebhookReady(namespace string) error {
+	cmd := exec.Command("kubectl", "wait",
+		"--for=condition=Ready",
+		"pod", "-l", "control-plane=controller-manager",
+		"-n", namespace,
+		"--timeout=5s")
+	_, err := Run(cmd)
+	if err != nil {
+		return fmt.Errorf("controller pod not yet Ready: %w", err)
+	}
+	return nil
+}