Merge pull request #5055 from hjiawei/fluent-bit-syslog-ca-bundle

logcollector: give fluent-bit its own trusted bundle; user CA for Splunk HEC

Author: hjiawei

pkg/controller/logcollector/logcollector_controller.go

@@ -142,6 +142,14 @@ func add(mgr manager.Manager, c ctrlruntime.Controller) error {
 		return fmt.Errorf("logcollector-controller failed to watch ConfigMap %s: %v", rlogcollector.FluentBitFilterConfigMapName, err)
 	}
 
+	// Watch the user-supplied CA ConfigMaps so creating or rotating a syslog or
+	// Splunk CA takes effect without waiting for an unrelated reconcile.
+	for _, caConfigMap := range []string{rlogcollector.SyslogCAConfigMapName, rlogcollector.SplunkCAConfigMapName} {
+		if err = utils.AddConfigMapWatch(c, caConfigMap, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil {
+			return fmt.Errorf("logcollector-controller failed to watch ConfigMap %s: %v", caConfigMap, err)
+		}
+	}
+
 	// Watch the rendered configuration ConfigMaps so tampering with them
 	// triggers a reconcile that restores the rendered content.
 	for _, configMapName := range []string{
@@ -465,11 +473,11 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile
 	}
 
 	// Fluent Bit needs to mount system certificates in the case where Splunk, Syslog or AWS are used.
-	trustedBundle, err := certificateManager.CreateTrustedBundleWithSystemRootCertificates(prometheusCertificate, linseedCertificate)
-	if err != nil {
-		r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create tigera-ca-bundle configmap", err, reqLogger)
-		return reconcile.Result{}, err
-	}
+	// The bundle carries fluent-bit's own name: calico-system's shared tigera-ca-bundle is rendered by
+	// the core Installation controller with a different certificate set, and the component handler
+	// replaces ConfigMap data wholesale — an unnamed bundle here would fight it, and additions like
+	// the syslog user CA would be lost to whichever controller wrote last.
+	trustedBundle := certificatemanagement.CreateNamedTrustedBundle(render.FluentBitNodeName, certificateManager.KeyPair(), true, prometheusCertificate, linseedCertificate)
 
 	certificateManager.AddToStatusManager(r.status, render.LogCollectorNamespace)
 
@@ -524,7 +532,7 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile
 	var useSyslogCertificate bool
 	if instance.Spec.AdditionalStores != nil {
 		if instance.Spec.AdditionalStores.Syslog != nil && instance.Spec.AdditionalStores.Syslog.Encryption == operatorv1.EncryptionTLS {
-			syslogCert, err := getSysLogCertificate(r.client)
+			syslogCert, err := getUserCACertificate(r.client, rlogcollector.SyslogCAConfigMapName)
 			if err != nil {
 				r.status.SetDegraded(operatorv1.ResourceReadError, "Error loading Syslog certificate", err, reqLogger)
 				return reconcile.Result{}, err
@@ -534,6 +542,22 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile
 				trustedBundle.AddCertificates(syslogCert)
 			}
 		}
+		// The Splunk output verifies https HEC endpoints against the trusted
+		// bundle, so a user CA for a self-hosted Splunk rides the same way as
+		// the syslog one. Plain-http endpoints do no verification, so like
+		// syslog's TLS gate the CA is only loaded for https.
+		if splunk := instance.Spec.AdditionalStores.Splunk; splunk != nil {
+			if proto, _, _, err := url.ParseEndpoint(splunk.Endpoint); err == nil && proto == "https" {
+				splunkCert, err := getUserCACertificate(r.client, rlogcollector.SplunkCAConfigMapName)
+				if err != nil {
+					r.status.SetDegraded(operatorv1.ResourceReadError, "Error loading Splunk certificate", err, reqLogger)
+					return reconcile.Result{}, err
+				}
+				if splunkCert != nil {
+					trustedBundle.AddCertificates(splunkCert)
+				}
+			}
+		}
 	}
 
 	if instance.Spec.AdditionalStores != nil {
@@ -872,24 +896,25 @@ func getEksCloudwatchLogConfig(client client.Client, interval int32, region, gro
 	}, nil
 }
 
-func getSysLogCertificate(client client.Client) (certificatemanagement.CertificateInterface, error) {
+// getUserCACertificate reads an optional user-supplied CA from the named
+// ConfigMap in the operator namespace (key tls.crt), for stores whose TLS
+// endpoint is not signed by a publicly trusted CA.
+func getUserCACertificate(client client.Client, name string) (certificatemanagement.CertificateInterface, error) {
 	cm := &corev1.ConfigMap{}
 	cmNamespacedName := types.NamespacedName{
-		Name:      rlogcollector.SyslogCAConfigMapName,
+		Name:      name,
 		Namespace: common.OperatorNamespace(),
 	}
 	if err := client.Get(context.Background(), cmNamespacedName, cm); err != nil {
 		if errors.IsNotFound(err) {
-			log.Info(fmt.Sprintf("ConfigMap %q is not found, assuming syslog's certificate is signed by publicly trusted CA", rlogcollector.SyslogCAConfigMapName))
+			log.Info(fmt.Sprintf("ConfigMap %q is not found, assuming the endpoint's certificate is signed by publicly trusted CA", name))
 			return nil, nil
 		}
-		return nil, fmt.Errorf("failed to read ConfigMap %q: %s", rlogcollector.SyslogCAConfigMapName, err)
+		return nil, fmt.Errorf("failed to read ConfigMap %q: %s", name, err)
 	}
 	if len(cm.Data[corev1.TLSCertKey]) == 0 {
-		log.Info(fmt.Sprintf("ConfigMap %q does not have a field named %q, assuming syslog's certificate is signed by publicly trusted CA", rlogcollector.SyslogCAConfigMapName, corev1.TLSCertKey))
+		log.Info(fmt.Sprintf("ConfigMap %q does not have a field named %q, assuming the endpoint's certificate is signed by publicly trusted CA", name, corev1.TLSCertKey))
 		return nil, nil
 	}
-	syslogCert := certificatemanagement.NewCertificate(rlogcollector.SyslogCAConfigMapName, common.OperatorNamespace(), []byte(cm.Data[corev1.TLSCertKey]), nil)
-
-	return syslogCert, nil
+	return certificatemanagement.NewCertificate(name, common.OperatorNamespace(), []byte(cm.Data[corev1.TLSCertKey]), nil), nil
 }

pkg/controller/logcollector/logcollector_controller_test.go

@@ -494,6 +494,49 @@ var _ = Describe("LogCollector controller tests", func() {
 				Expect(conf).To(ContainSubstring(`"splunk_token": "${SPLUNK_HEC_TOKEN}"`))
 			})
 
+			It("renders a user-supplied Splunk CA into fluent-bit's bundle", func() {
+				caPEM := "-----BEGIN CERTIFICATE-----\nsplunk-user-ca\n-----END CERTIFICATE-----"
+				Expect(c.Create(ctx, &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Name: rlogcollector.SplunkCAConfigMapName, Namespace: common.OperatorNamespace()},
+					Data:       map[string]string{corev1.TLSCertKey: caPEM},
+				})).NotTo(HaveOccurred())
+
+				_, err := r.Reconcile(ctx, reconcile.Request{})
+				Expect(err).ShouldNot(HaveOccurred())
+
+				bundle := corev1.ConfigMap{
+					TypeMeta:   metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-ca-bundle-system-certs", Namespace: render.LogCollectorNamespace},
+				}
+				Expect(test.GetResource(c, &bundle)).To(BeNil())
+				Expect(bundle.Data["tigera-ca-bundle.crt"]).To(ContainSubstring(caPEM))
+			})
+
+			It("does not load the Splunk CA for a plain-http endpoint", func() {
+				lc := operatorv1.LogCollector{
+					TypeMeta:   metav1.TypeMeta{Kind: "LogCollector", APIVersion: "operator.tigera.io/v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"},
+				}
+				Expect(test.GetResource(c, &lc)).To(BeNil())
+				lc.Spec.AdditionalStores.Splunk.Endpoint = "http://localhost:1234"
+				Expect(c.Update(ctx, &lc)).NotTo(HaveOccurred())
+				caPEM := "-----BEGIN CERTIFICATE-----\nsplunk-user-ca\n-----END CERTIFICATE-----"
+				Expect(c.Create(ctx, &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Name: rlogcollector.SplunkCAConfigMapName, Namespace: common.OperatorNamespace()},
+					Data:       map[string]string{corev1.TLSCertKey: caPEM},
+				})).NotTo(HaveOccurred())
+
+				_, err := r.Reconcile(ctx, reconcile.Request{})
+				Expect(err).ShouldNot(HaveOccurred())
+
+				bundle := corev1.ConfigMap{
+					TypeMeta:   metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-ca-bundle-system-certs", Namespace: render.LogCollectorNamespace},
+				}
+				Expect(test.GetResource(c, &bundle)).To(BeNil())
+				Expect(bundle.Data["tigera-ca-bundle.crt"]).NotTo(ContainSubstring(caPEM))
+			})
+
 			Context("Disable feature via license", func() {
 				BeforeEach(func() {
 					By("Deleting the previous license")
@@ -584,6 +627,42 @@ var _ = Describe("LogCollector controller tests", func() {
 				Expect(conf).To(ContainSubstring(`"call": "syslog_pack"`))
 			})
 
+			It("renders the syslog user CA into fluent-bit's own bundle, not the shared tigera-ca-bundle", func() {
+				By("Switching the syslog store to TLS with a user-supplied CA")
+				lc := operatorv1.LogCollector{
+					TypeMeta:   metav1.TypeMeta{Kind: "LogCollector", APIVersion: "operator.tigera.io/v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"},
+				}
+				Expect(test.GetResource(c, &lc)).To(BeNil())
+				lc.Spec.AdditionalStores.Syslog.Encryption = operatorv1.EncryptionTLS
+				Expect(c.Update(ctx, &lc)).NotTo(HaveOccurred())
+				caPEM := "-----BEGIN CERTIFICATE-----\nsyslog-user-ca\n-----END CERTIFICATE-----"
+				Expect(c.Create(ctx, &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Name: rlogcollector.SyslogCAConfigMapName, Namespace: common.OperatorNamespace()},
+					Data:       map[string]string{corev1.TLSCertKey: caPEM},
+				})).NotTo(HaveOccurred())
+
+				_, err := r.Reconcile(ctx, reconcile.Request{})
+				Expect(err).ShouldNot(HaveOccurred())
+
+				// The bundle must be fluent-bit's own: the core Installation controller
+				// renders calico-system's shared tigera-ca-bundle with a different
+				// certificate set, so additions made there would be overwritten.
+				bundle := corev1.ConfigMap{
+					TypeMeta:   metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "calico-fluent-bit-ca-bundle-system-certs", Namespace: render.LogCollectorNamespace},
+				}
+				Expect(test.GetResource(c, &bundle)).To(BeNil())
+				Expect(bundle.Data["tigera-ca-bundle.crt"]).To(ContainSubstring(caPEM))
+
+				shared := corev1.ConfigMap{
+					TypeMeta:   metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
+					ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: render.LogCollectorNamespace},
+				}
+				Expect(errors.IsNotFound(test.GetResource(c, &shared))).To(BeTrue(),
+					"the logcollector controller must not render the shared tigera-ca-bundle")
+			})
+
 			Context("Disable feature via license", func() {
 				BeforeEach(func() {
 					By("Deleting the previous license")

pkg/render/logcollector/fluentbit_test.go

@@ -49,6 +49,7 @@ import (
 	"github.com/tigera/operator/pkg/render/logcollector"
 	"github.com/tigera/operator/pkg/render/testutils"
 	"github.com/tigera/operator/pkg/tls"
+	"github.com/tigera/operator/pkg/tls/certificatemanagement"
 	"github.com/tigera/operator/test"
 )
 
@@ -82,7 +83,7 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() {
 			},
 			FluentBitKeyPair:       metricsSecret,
 			EKSLogForwarderKeyPair: eksSecret,
-			TrustedBundle:          certificateManager.CreateTrustedBundle(),
+			TrustedBundle:          certificatemanagement.CreateNamedTrustedBundle(render.FluentBitNodeName, certificateManager.KeyPair(), true),
 		}
 	})
 
@@ -693,7 +694,7 @@ var _ = Describe("Tigera Secure Fluent Bit rendering tests", func() {
 		for _, vol := range ds.Spec.Template.Spec.Volumes {
 			volnames = append(volnames, vol.Name)
 		}
-		Expect(volnames).To(ContainElement("tigera-ca-bundle"))
+		Expect(volnames).To(ContainElement("calico-fluent-bit-ca-bundle-system-certs"))
 
 		// Syslog TLS configuration is in the ConfigMap.
 		cm := rtest.GetResource(resources, logcollector.FluentBitConfConfigMapName, render.LogCollectorNamespace, "", "v1", "ConfigMap").(*corev1.ConfigMap)

pkg/render/logcollector/logcollector.go

@@ -73,6 +73,7 @@ const (
 	SysLogPublicCertKey                      = "ca-bundle.crt"
 	SysLogPublicCAPath                       = SysLogPublicCADir + SysLogPublicCertKey
 	SyslogCAConfigMapName                    = "syslog-ca"
+	SplunkCAConfigMapName                    = "splunk-ca"
 
 	// Constants for Linseed token volume mounting in managed clusters.
 	LinseedTokenVolumeName = render.LinseedTokenVolumeName

pkg/render/logcollector/rendered_config_test.go

@@ -35,6 +35,7 @@ import (
 	rmeta "github.com/tigera/operator/pkg/render/common/meta"
 	rtest "github.com/tigera/operator/pkg/render/common/test"
 	"github.com/tigera/operator/pkg/render/logcollector"
+	"github.com/tigera/operator/pkg/tls/certificatemanagement"
 )
 
 // TestRenderedConfigGoldens pins the full fluent-bit configuration the
@@ -336,7 +337,7 @@ func goldenBaseConfig(t *testing.T) *logcollector.FluentBitConfiguration {
 		},
 		FluentBitKeyPair:       metricsSecret,
 		EKSLogForwarderKeyPair: eksSecret,
-		TrustedBundle:          certificateManager.CreateTrustedBundle(),
+		TrustedBundle:          certificatemanagement.CreateNamedTrustedBundle(render.FluentBitNodeName, certificateManager.KeyPair(), true),
 	}
 }