logcollector: user CA for Splunk HEC, watch the CA ConfigMaps

The Splunk output verifies https HEC endpoints against the trusted
bundle, but nothing put a private CA there — a self-hosted Splunk with a
self-signed certificate could not be used since the splunk-public-
certificate secret was removed (#2545). Accept one the same way syslog
does: a splunk-ca ConfigMap in tigera-operator (key tls.crt), folded
into fluent-bit's bundle.

Also watch the syslog-ca and splunk-ca ConfigMaps: previously a created
or rotated CA sat unnoticed until an unrelated event triggered a
reconcile.

Validated on a live cluster: the Splunk e2e forwards flow logs to an
https HEC endpoint serving Splunk's self-signed certificate.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: hjiawei

pkg/controller/logcollector/logcollector_controller.go

@@ -142,6 +142,14 @@ func add(mgr manager.Manager, c ctrlruntime.Controller) error {
 		return fmt.Errorf("logcollector-controller failed to watch ConfigMap %s: %v", rlogcollector.FluentBitFilterConfigMapName, err)
 	}
 
+	// Watch the user-supplied CA ConfigMaps so creating or rotating a syslog or
+	// Splunk CA takes effect without waiting for an unrelated reconcile.
+	for _, caConfigMap := range []string{rlogcollector.SyslogCAConfigMapName, rlogcollector.SplunkCAConfigMapName} {
+		if err = utils.AddConfigMapWatch(c, caConfigMap, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil {
+			return fmt.Errorf("logcollector-controller failed to watch ConfigMap %s: %v", caConfigMap, err)
+		}
+	}
+
 	// Watch the rendered configuration ConfigMaps so tampering with them
 	// triggers a reconcile that restores the rendered content.
 	for _, configMapName := range []string{
@@ -524,7 +532,7 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile
 	var useSyslogCertificate bool
 	if instance.Spec.AdditionalStores != nil {
 		if instance.Spec.AdditionalStores.Syslog != nil && instance.Spec.AdditionalStores.Syslog.Encryption == operatorv1.EncryptionTLS {
-			syslogCert, err := getSysLogCertificate(r.client)
+			syslogCert, err := getUserCACertificate(r.client, rlogcollector.SyslogCAConfigMapName)
 			if err != nil {
 				r.status.SetDegraded(operatorv1.ResourceReadError, "Error loading Syslog certificate", err, reqLogger)
 				return reconcile.Result{}, err
@@ -534,6 +542,22 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile
 				trustedBundle.AddCertificates(syslogCert)
 			}
 		}
+		// The Splunk output verifies https HEC endpoints against the trusted
+		// bundle, so a user CA for a self-hosted Splunk rides the same way as
+		// the syslog one. Plain-http endpoints do no verification, so like
+		// syslog's TLS gate the CA is only loaded for https.
+		if splunk := instance.Spec.AdditionalStores.Splunk; splunk != nil {
+			if proto, _, _, err := url.ParseEndpoint(splunk.Endpoint); err == nil && proto == "https" {
+				splunkCert, err := getUserCACertificate(r.client, rlogcollector.SplunkCAConfigMapName)
+				if err != nil {
+					r.status.SetDegraded(operatorv1.ResourceReadError, "Error loading Splunk certificate", err, reqLogger)
+					return reconcile.Result{}, err
+				}
+				if splunkCert != nil {
+					trustedBundle.AddCertificates(splunkCert)
+				}
+			}
+		}
 	}
 
 	if instance.Spec.AdditionalStores != nil {
@@ -872,24 +896,25 @@ func getEksCloudwatchLogConfig(client client.Client, interval int32, region, gro
 	}, nil
 }
 
-func getSysLogCertificate(client client.Client) (certificatemanagement.CertificateInterface, error) {
+// getUserCACertificate reads an optional user-supplied CA from the named
+// ConfigMap in the operator namespace (key tls.crt), for stores whose TLS
+// endpoint is not signed by a publicly trusted CA.
+func getUserCACertificate(client client.Client, name string) (certificatemanagement.CertificateInterface, error) {
 	cm := &corev1.ConfigMap{}
 	cmNamespacedName := types.NamespacedName{
-		Name:      rlogcollector.SyslogCAConfigMapName,
+		Name:      name,
 		Namespace: common.OperatorNamespace(),
 	}
 	if err := client.Get(context.Background(), cmNamespacedName, cm); err != nil {
 		if errors.IsNotFound(err) {
-			log.Info(fmt.Sprintf("ConfigMap %q is not found, assuming syslog's certificate is signed by publicly trusted CA", rlogcollector.SyslogCAConfigMapName))
+			log.Info(fmt.Sprintf("ConfigMap %q is not found, assuming the endpoint's certificate is signed by publicly trusted CA", name))
 			return nil, nil
 		}
-		return nil, fmt.Errorf("failed to read ConfigMap %q: %s", rlogcollector.SyslogCAConfigMapName, err)
+		return nil, fmt.Errorf("failed to read ConfigMap %q: %s", name, err)
 	}
 	if len(cm.Data[corev1.TLSCertKey]) == 0 {
-		log.Info(fmt.Sprintf("ConfigMap %q does not have a field named %q, assuming syslog's certificate is signed by publicly trusted CA", rlogcollector.SyslogCAConfigMapName, corev1.TLSCertKey))
+		log.Info(fmt.Sprintf("ConfigMap %q does not have a field named %q, assuming the endpoint's certificate is signed by publicly trusted CA", name, corev1.TLSCertKey))
 		return nil, nil
 	}
-	syslogCert := certificatemanagement.NewCertificate(rlogcollector.SyslogCAConfigMapName, common.OperatorNamespace(), []byte(cm.Data[corev1.TLSCertKey]), nil)
-
-	return syslogCert, nil
+	return certificatemanagement.NewCertificate(name, common.OperatorNamespace(), []byte(cm.Data[corev1.TLSCertKey]), nil), nil
 }

pkg/controller/logcollector/logcollector_controller_test.go

@@ -494,6 +494,49 @@ var _ = Describe("LogCollector controller tests", func() {
 				Expect(conf).To(ContainSubstring(`"splunk_token": "${SPLUNK_HEC_TOKEN}"`))
 			})
 
+			It("renders a user-supplied Splunk CA into fluent-bit's bundle", func() {
+				caPEM := "-----BEGIN CERTIFICATE-----\nsplunk-user-ca\n-----END CERTIFICATE-----"
+				Expect(c.Create(ctx, &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Name: rlogcollector.SplunkCAConfigMapName, Namespace: common.OperatorNamespace()},
+					Data:       map[string]string{corev1.TLSCertKey: caPEM},
+				})).NotTo(HaveOccurred())
+
+				_, err := r.Reconcile(ctx, reconcile.Request{})
+				Expect(err).ShouldNot(HaveOccurred())
+
+				bundle := corev1.ConfigMap{
+					TypeMeta:   metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-ca-bundle-system-certs", Namespace: render.LogCollectorNamespace},
+				}
+				Expect(test.GetResource(c, &bundle)).To(BeNil())
+				Expect(bundle.Data["tigera-ca-bundle.crt"]).To(ContainSubstring(caPEM))
+			})
+
+			It("does not load the Splunk CA for a plain-http endpoint", func() {
+				lc := operatorv1.LogCollector{
+					TypeMeta:   metav1.TypeMeta{Kind: "LogCollector", APIVersion: "operator.tigera.io/v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"},
+				}
+				Expect(test.GetResource(c, &lc)).To(BeNil())
+				lc.Spec.AdditionalStores.Splunk.Endpoint = "http://localhost:1234"
+				Expect(c.Update(ctx, &lc)).NotTo(HaveOccurred())
+				caPEM := "-----BEGIN CERTIFICATE-----\nsplunk-user-ca\n-----END CERTIFICATE-----"
+				Expect(c.Create(ctx, &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Name: rlogcollector.SplunkCAConfigMapName, Namespace: common.OperatorNamespace()},
+					Data:       map[string]string{corev1.TLSCertKey: caPEM},
+				})).NotTo(HaveOccurred())
+
+				_, err := r.Reconcile(ctx, reconcile.Request{})
+				Expect(err).ShouldNot(HaveOccurred())
+
+				bundle := corev1.ConfigMap{
+					TypeMeta:   metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-ca-bundle-system-certs", Namespace: render.LogCollectorNamespace},
+				}
+				Expect(test.GetResource(c, &bundle)).To(BeNil())
+				Expect(bundle.Data["tigera-ca-bundle.crt"]).NotTo(ContainSubstring(caPEM))
+			})
+
 			Context("Disable feature via license", func() {
 				BeforeEach(func() {
 					By("Deleting the previous license")

pkg/render/logcollector/logcollector.go

@@ -73,6 +73,7 @@ const (
 	SysLogPublicCertKey                      = "ca-bundle.crt"
 	SysLogPublicCAPath                       = SysLogPublicCADir + SysLogPublicCertKey
 	SyslogCAConfigMapName                    = "syslog-ca"
+	SplunkCAConfigMapName                    = "splunk-ca"
 
 	// Constants for Linseed token volume mounting in managed clusters.
 	LinseedTokenVolumeName = render.LinseedTokenVolumeName