fix: use shared system trust store sds secret (#9357)

* use shared system trust store sds secret

Signed-off-by: Guy Daich <guy.daich@sap.com>

Author: guydc

api/v1alpha1/envoygateway_types.go

@@ -162,7 +162,7 @@ type GatewayAPISettings struct {
 // RuntimeFlag defines a runtime flag used to guard breaking changes or risky experimental features in new Envoy Gateway releases.
 // A runtime flag may be enabled or disabled by default and can be toggled through the EnvoyGateway resource.
 // +enum
-// +kubebuilder:validation:Enum=XDSNameSchemeV2;EndpointSliceIndex
+// +kubebuilder:validation:Enum=XDSNameSchemeV2;EndpointSliceIndex;PerResourceSystemCASecret
 type RuntimeFlag string
 
 const (
@@ -175,6 +175,14 @@ const (
 	// If the additional controller memory usage for the indexes becomes a concern,
 	// consider disabling this flag.
 	EndpointSliceIndex RuntimeFlag = "EndpointSliceIndex"
+
+	// PerResourceSystemCASecret restores the pre-1.x behavior of emitting one SDS secret per
+	// BackendTLSPolicy or Backend resource that uses WellKnownCACertificates: System, instead
+	// of sharing a single system_ca_certificates secret across all of them.
+	// Disabled by default (i.e. the shared secret is used). Enable this flag to opt out during
+	// upgrades — Envoy must warm the new system_ca_certificates secret before clusters can use
+	// it, which may cause a brief disruption to new connections on first enable.
+	PerResourceSystemCASecret RuntimeFlag = "PerResourceSystemCASecret" //nolint:gosec // not a credential
 )
 
 // RuntimeFlags provide a mechanism to guard breaking changes or risky experimental features in new Envoy Gateway releases.

internal/gatewayapi/backendtlspolicy.go

@@ -192,6 +192,11 @@ func mergeServerValidationTLSConfigs(
 
 	if btpValidationTLSConfig.CACertificate != nil {
 		mergedConfig.CACertificate = btpValidationTLSConfig.CACertificate
+		// If the BTLSP provides an explicit CA certificate, it overrides the backend's
+		// system trust store — clear UseSystemTrustStore so the IR is consistent.
+		if !btpValidationTLSConfig.UseSystemTrustStore {
+			mergedConfig.UseSystemTrustStore = false
+		}
 	}
 	if btpValidationTLSConfig.SNI != nil { // BTP takes precedence for SNI, if set, it will override Backend resource SNI and disable AutoSNIFromEndpointHostname
 		mergedConfig.SNI = btpValidationTLSConfig.SNI
@@ -276,8 +281,12 @@ func (t *Translator) processServerValidationTLSSettings(
 	if !tlsConfig.InsecureSkipVerify {
 		tlsConfig.UseSystemTrustStore = ptr.Deref(backend.Spec.TLS.WellKnownCACertificates, "") == gwapiv1.WellKnownCACertificatesSystem
 		if tlsConfig.UseSystemTrustStore {
+			name := fmt.Sprintf("%s/%s-ca", backend.Name, backend.Namespace)
+			if !t.PerResourceSystemCASecret {
+				name = ir.SystemTrustStoreSecretName
+			}
 			tlsConfig.CACertificate = &ir.TLSCACertificate{
-				Name: fmt.Sprintf("%s/%s-ca", backend.Name, backend.Namespace),
+				Name: name,
 			}
 		} else if len(backend.Spec.TLS.CACertificateRefs) > 0 {
 			caRefs := getObjectReferences(gwapiv1.Namespace(backend.Namespace), backend.Spec.TLS.CACertificateRefs)
@@ -504,8 +513,12 @@ func (t *Translator) getBackendTLSBundle(backendTLSPolicy *gwapiv1.BackendTLSPol
 		SubjectAltNames:     subjectAltNames,
 	}
 	if tlsBundle.UseSystemTrustStore {
+		name := fmt.Sprintf("%s/%s-ca", backendTLSPolicy.Name, backendTLSPolicy.Namespace)
+		if !t.PerResourceSystemCASecret {
+			name = ir.SystemTrustStoreSecretName
+		}
 		tlsBundle.CACertificate = &ir.TLSCACertificate{
-			Name: fmt.Sprintf("%s/%s-ca", backendTLSPolicy.Name, backendTLSPolicy.Namespace),
+			Name: name,
 		}
 		return tlsBundle, nil
 	}

internal/gatewayapi/listener_test.go

@@ -1433,7 +1433,7 @@ func TestProcessBackendRefsBackendTLSPolicy(t *testing.T) {
 	backendMetadata := &ir.ResourceMetadata{Name: backendName, Namespace: ns}
 	backendPolicyTLS := &ir.TLSUpstreamConfig{
 		SNI: new("otel.example.com"), UseSystemTrustStore: true,
-		CACertificate: &ir.TLSCACertificate{Name: "otel-tls/test-ns-ca"}, SubjectAltNames: []ir.SubjectAltName{},
+		CACertificate: &ir.TLSCACertificate{Name: ir.SystemTrustStoreSecretName}, SubjectAltNames: []ir.SubjectAltName{},
 		TLSConfig: ir.TLSConfig{MinVersion: new(ir.TLSv12), MaxVersion: new(ir.TLSv13)},
 	}
 
@@ -1472,7 +1472,7 @@ func TestProcessBackendRefsBackendTLSPolicy(t *testing.T) {
 	serviceMetadata := &ir.ResourceMetadata{Name: serviceName, Namespace: ns, SectionName: "4317"}
 	servicePolicyTLS := &ir.TLSUpstreamConfig{
 		SNI: new("otel-svc.example.com"), UseSystemTrustStore: true,
-		CACertificate: &ir.TLSCACertificate{Name: "otel-svc-tls/test-ns-ca"}, SubjectAltNames: []ir.SubjectAltName{},
+		CACertificate: &ir.TLSCACertificate{Name: ir.SystemTrustStoreSecretName}, SubjectAltNames: []ir.SubjectAltName{},
 		TLSConfig: ir.TLSConfig{MinVersion: new(ir.TLSv12), MaxVersion: new(ir.TLSv13)},
 	}
 

internal/gatewayapi/runner/runner.go

@@ -299,6 +299,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
 					ControllerNamespace:             r.ControllerNamespace,
 					GatewayNamespaceMode:            r.EnvoyGateway.GatewayNamespaceMode(),
 					MergeGateways:                   gatewayapi.IsMergeGatewaysEnabled(resources),
+					PerResourceSystemCASecret:       r.EnvoyGateway.RuntimeFlags.IsEnabled(egv1a1.PerResourceSystemCASecret),
 					WasmCache:                       r.wasmCache,
 					RunningOnHost:                   r.EnvoyGateway.Provider != nil && r.EnvoyGateway.Provider.IsRunningOnHost(),
 					InfraRemotelyManaged:            r.EnvoyGateway.Provider != nil && r.EnvoyGateway.Provider.IsInfraManagedRemotely(),

internal/gatewayapi/testdata/backend-system-truststore-btlsp-override-per-resource-secret.in.yaml

@@ -0,0 +1,142 @@
+# Two scenarios tested together:
+# 1. Backend with wellKnownCACertificates: System overridden by a BackendTLSPolicy
+#    with an explicit caCertificateRefs — BTLSP's CA cert must win, UseSystemTrustStore cleared.
+# 2. Backend with explicit caCertificateRefs overridden by a BackendTLSPolicy
+#    with wellKnownCACertificates: System — system trust store must win.
+gateways:
+  - apiVersion: gateway.networking.k8s.io/v1
+    kind: Gateway
+    metadata:
+      name: gateway-btls
+      namespace: envoy-gateway
+    spec:
+      gatewayClassName: envoy-gateway-class
+      listeners:
+        - name: http
+          protocol: HTTP
+          port: 80
+          allowedRoutes:
+            namespaces:
+              from: All
+httpRoutes:
+  - apiVersion: gateway.networking.k8s.io/v1
+    kind: HTTPRoute
+    metadata:
+      name: httproute-btls
+      namespace: envoy-gateway
+    spec:
+      parentRefs:
+        - namespace: envoy-gateway
+          name: gateway-btls
+          sectionName: http
+      rules:
+        - matches:
+            - path:
+                type: Exact
+                value: /exact
+          backendRefs:
+            - group: gateway.envoyproxy.io
+              kind: Backend
+              name: backend-system-ca
+              namespace: envoy-gateway
+              port: 8080
+        - matches:
+            - path:
+                type: Exact
+                value: /exact2
+          backendRefs:
+            - group: gateway.envoyproxy.io
+              kind: Backend
+              name: backend-inline-ca
+              namespace: envoy-gateway
+              port: 8080
+backends:
+  - apiVersion: gateway.envoyproxy.io/v1alpha1
+    kind: Backend
+    metadata:
+      name: backend-system-ca
+      namespace: envoy-gateway
+    spec:
+      endpoints:
+        - fqdn:
+            hostname: example.com
+            port: 8080
+      tls:
+        # Scenario 1: Backend uses system trust store, BTLSP overrides with inline CA.
+        wellKnownCACertificates: System
+  - apiVersion: gateway.envoyproxy.io/v1alpha1
+    kind: Backend
+    metadata:
+      name: backend-inline-ca
+      namespace: envoy-gateway
+    spec:
+      endpoints:
+        - fqdn:
+            hostname: example2.com
+            port: 8080
+      tls:
+        # Scenario 2: Backend uses inline CA, BTLSP overrides with system trust store.
+        caCertificateRefs:
+          - name: ca-cmap
+            group: ""
+            kind: ConfigMap
+backendTLSPolicies:
+  - apiVersion: gateway.networking.k8s.io/v1alpha2
+    kind: BackendTLSPolicy
+    metadata:
+      name: policy-btls-inline-override
+      namespace: envoy-gateway
+    spec:
+      targetRefs:
+        - group: gateway.envoyproxy.io
+          kind: Backend
+          name: backend-system-ca
+      validation:
+        # Scenario 1: BTLSP overrides with explicit CA cert — must win over system trust store.
+        caCertificateRefs:
+          - name: ca-cmap
+            group: ""
+            kind: ConfigMap
+        hostname: example.com
+  - apiVersion: gateway.networking.k8s.io/v1alpha2
+    kind: BackendTLSPolicy
+    metadata:
+      name: policy-btls-system-override
+      namespace: envoy-gateway
+    spec:
+      targetRefs:
+        - group: gateway.envoyproxy.io
+          kind: Backend
+          name: backend-inline-ca
+      validation:
+        # Scenario 2: BTLSP overrides with system trust store — must win over inline CA.
+        wellKnownCACertificates: System
+        hostname: example2.com
+configMaps:
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: ca-cmap
+      namespace: envoy-gateway
+    data:
+      ca.crt: |
+        -----BEGIN CERTIFICATE-----
+        MIIDJzCCAg+gAwIBAgIUAl6UKIuKmzte81cllz5PfdN2IlIwDQYJKoZIhvcNAQEL
+        BQAwIzEQMA4GA1UEAwwHbXljaWVudDEPMA0GA1UECgwGa3ViZWRiMB4XDTIzMTAw
+        MjA1NDE1N1oXDTI0MTAwMTA1NDE1N1owIzEQMA4GA1UEAwwHbXljaWVudDEPMA0G
+        A1UECgwGa3ViZWRiMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwSTc
+        1yj8HW62nynkFbXo4VXKv2jC0PM7dPVky87FweZcTKLoWQVPQE2p2kLDK6OEszmM
+        yyr+xxWtyiveremrWqnKkNTYhLfYPhgQkczib7eUalmFjUbhWdLvHakbEgCodn3b
+        kz57mInX2VpiDOKg4kyHfiuXWpiBqrCx0KNLpxo3DEQcFcsQTeTHzh4752GV04RU
+        Ti/GEWyzIsl4Rg7tGtAwmcIPgUNUfY2Q390FGqdH4ahn+mw/6aFbW31W63d9YJVq
+        ioyOVcaMIpM5B/c7Qc8SuhCI1YGhUyg4cRHLEw5VtikioyE3X04kna3jQAj54YbR
+        bpEhc35apKLB21HOUQIDAQABo1MwUTAdBgNVHQ4EFgQUyvl0VI5vJVSuYFXu7B48
+        6PbMEAowHwYDVR0jBBgwFoAUyvl0VI5vJVSuYFXu7B486PbMEAowDwYDVR0TAQH/
+        BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAMLxrgFVMuNRq2wAwcBt7SnNR5Cfz
+        2MvXq5EUmuawIUi9kaYjwdViDREGSjk7JW17vl576HjDkdfRwi4E28SydRInZf6J
+        i8HZcZ7caH6DxR335fgHVzLi5NiTce/OjNBQzQ2MJXVDd8DBmG5fyatJiOJQ4bWE
+        A7FlP0RdP3CO3GWE0M5iXOB2m1qWkE2eyO4UHvwTqNQLdrdAXgDQlbam9e4BG3Gg
+        d/6thAkWDbt/QNT+EJHDCvhDRKh1RuGHyg+Y+/nebTWWrFWsktRrbOoHCZiCpXI1
+        3eXE6nt0YkgtDxG22KqnhpAg9gUSs2hlhoxyvkzyF0mu6NhPlwAgnq7+/Q==
+        -----END CERTIFICATE-----
+