feat: promote native sidecars to GA and enable by default (#15267)

Advances [native sidecars](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/)
support from beta to GA, and makes them the default injection mode for
the proxy.

This requires Kubernetes v1.29 or later, which is below our current
minimum supported version (v1.31). The cluster must also have the
`SidecarContainers` feature gate enabled, which is the default in all
recent Kubernetes versions.

You can verify the feature gate is enabled with:
```
$ kubectl get --raw /metrics | grep feature.*SidecarContainers
kubernetes_feature_enabled{name="SidecarContainers",stage=""} 1
```

Changes:
- `proxy.nativeSidecar` now defaults to `true`
- The `config.beta.linkerd.io/proxy-enable-native-sidecar` annotation is
deprecated in favor of `config.linkerd.io/proxy-enable-native-sidecar`

Author: alpeb

charts/linkerd-control-plane/values.yaml

@@ -228,7 +228,7 @@ proxy:
   # Kubernetes but no longer than the pod's `terminationGracePeriodSeconds`.
   # See [Lifecycle
   # hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks)
-  # for more info on container lifecycle hooks.
+  # for more info on container lifecycle hooks. Ignored when nativeSidecar=true.
   waitBeforeExitSeconds: 0
   # -- If set, the application container will not start until the proxy is
   # ready
@@ -251,10 +251,8 @@ proxy:
   defaultInboundPolicy: "all-unauthenticated"
   # -- Configures the outbound transport mode. Valid values are "transport-header" and "transparent"
   outboundTransportMode: transport-header
-  # -- Enable KEP-753 native sidecars
-  # This is a beta feature. It requires Kubernetes >= 1.29.
-  # If enabled, .proxy.waitBeforeExitSeconds should not be used.
-  nativeSidecar: false
+  # -- Enable [native sidecars](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/)
+  nativeSidecar: true
   # -- Native sidecar proxy startup probe parameters.
   # -- LivenessProbe timeout and delay configuration
   livenessProbe:

charts/partials/templates/_proxy.tpl

@@ -1,7 +1,4 @@
 {{ define "partials.proxy" -}}
-{{ if and .Values.proxy.nativeSidecar .Values.proxy.waitBeforeExitSeconds }}
-{{ fail "proxy.nativeSidecar and waitBeforeExitSeconds cannot be used simultaneously" }}
-{{- end }}
 {{- if not (has .Values.proxy.logHTTPHeaders (list "insecure" "off" "")) }}
 {{- fail "logHTTPHeaders must be one of: insecure | off" }}
 {{- end }}

cli/cmd/doc.go

@@ -282,11 +282,15 @@ func generateAnnotationsDocs() []annotationDoc {
 		},
 		{
 			Name:        k8s.ProxyEnableNativeSidecarAnnotationAlpha,
-			Description: "Enable KEP-753 native sidecars. This is a beta feature. It requires Kubernetes >= 1.29. If enabled, .proxy.waitBeforeExitSeconds should not be used. Deprecated in favor of " + k8s.ProxyEnableNativeSidecarAnnotationBeta,
+			Description: "Enable KEP-753 native sidecars. Deprecated in favor of " + k8s.ProxyEnableNativeSidecarAnnotation,
 		},
 		{
 			Name:        k8s.ProxyEnableNativeSidecarAnnotationBeta,
-			Description: "Enable KEP-753 native sidecars. This is a beta feature. It requires Kubernetes >= 1.29. If enabled, .proxy.waitBeforeExitSeconds should not be used.",
+			Description: "Enable KEP-753 native sidecars. Deprecated in favor of " + k8s.ProxyEnableNativeSidecarAnnotation,
+		},
+		{
+			Name:        k8s.ProxyEnableNativeSidecarAnnotation,
+			Description: "Enable KEP-753 native sidecars",
 		},
 		{
 			Name:        k8s.ProxyAdditionalEnvAnnotation,

cli/cmd/inject.go

@@ -498,7 +498,7 @@ func getOverrideAnnotations(values *linkerd2.Values, base *linkerd2.Values) map[
 	}
 
 	if proxy.NativeSidecar != baseProxy.NativeSidecar {
-		overrideAnnotations[k8s.ProxyEnableNativeSidecarAnnotationBeta] = strconv.FormatBool(proxy.NativeSidecar)
+		overrideAnnotations[k8s.ProxyEnableNativeSidecarAnnotation] = strconv.FormatBool(proxy.NativeSidecar)
 	}
 
 	return overrideAnnotations

cli/cmd/inject_test.go

@@ -725,7 +725,6 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
 	values.Proxy.Await = false
 	values.Proxy.AccessLog = "apache"
 	values.Proxy.ShutdownGracePeriod = "60s"
-	values.Proxy.NativeSidecar = true
 
 	expectedOverrides := map[string]string{
 		k8s.ProxyIgnoreInboundPortsAnnotation:  "8500-8505",
@@ -739,16 +738,15 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
 		k8s.ProxyLogLevelAnnotation:            "debug",
 		k8s.ProxyLogFormatAnnotation:           "cool",
 
-		k8s.ProxyEnableExternalProfilesAnnotation:  "true",
-		k8s.ProxyCPURequestAnnotation:              "10m",
-		k8s.ProxyCPULimitAnnotation:                "100m",
-		k8s.ProxyMemoryRequestAnnotation:           "10Mi",
-		k8s.ProxyMemoryLimitAnnotation:             "50Mi",
-		k8s.ProxyWaitBeforeExitSecondsAnnotation:   "10",
-		k8s.ProxyAwait:                             "disabled",
-		k8s.ProxyAccessLogAnnotation:               "apache",
-		k8s.ProxyShutdownGracePeriodAnnotation:     "60s",
-		k8s.ProxyEnableNativeSidecarAnnotationBeta: "true",
+		k8s.ProxyEnableExternalProfilesAnnotation: "true",
+		k8s.ProxyCPURequestAnnotation:             "10m",
+		k8s.ProxyCPULimitAnnotation:               "100m",
+		k8s.ProxyMemoryRequestAnnotation:          "10Mi",
+		k8s.ProxyMemoryLimitAnnotation:            "50Mi",
+		k8s.ProxyWaitBeforeExitSecondsAnnotation:  "10",
+		k8s.ProxyAwait:                            "disabled",
+		k8s.ProxyAccessLogAnnotation:              "apache",
+		k8s.ProxyShutdownGracePeriodAnnotation:    "60s",
 	}
 
 	overrides := getOverrideAnnotations(values, baseValues)
@@ -853,6 +851,7 @@ func TestOverwriteRegistry(t *testing.T) {
 }
 
 func diffOverrides(t *testing.T, expectedOverrides map[string]string, actualOverrides map[string]string) {
+	t.Helper()
 	if len(expectedOverrides) != len(actualOverrides) {
 		t.Fatalf("expected annotations:\n%s\nbut received:\n%s", expectedOverrides, actualOverrides)
 	}

cli/cmd/testdata/inject-filepath/expected/injected_nginx.yaml

@@ -19,6 +19,50 @@ spec:
         linkerd.io/workload-ns: ""
     spec:
       containers:
+      - image: nginx
+        name: nginx
+        ports:
+        - containerPort: 80
+          name: http
+      initContainers:
+      - args:
+        - --firewall-bin-path
+        - iptables-nft
+        - --firewall-save-bin-path
+        - iptables-nft-save
+        - --ipv6=false
+        - --incoming-proxy-port
+        - "4143"
+        - --outgoing-proxy-port
+        - "4140"
+        - --proxy-uid
+        - "2102"
+        - --inbound-ports-to-ignore
+        - 4190,4191,4567,4568
+        - --outbound-ports-to-ignore
+        - 4567,4568
+        command:
+        - /usr/lib/linkerd/linkerd2-proxy-init
+        image: cr.l5d.io/linkerd/proxy:install-proxy-version
+        imagePullPolicy: IfNotPresent
+        name: linkerd-init
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+          privileged: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 65534
+          runAsNonRoot: true
+          runAsUser: 65534
+          seccompProfile:
+            type: RuntimeDefault
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /run
+          name: linkerd-proxy-init-xtables-lock
       - env:
         - name: _pod_name
           valueFrom:
@@ -158,13 +202,6 @@ spec:
           value: linkerd-destination.linkerd.serviceaccount.identity.linkerd.cluster.local
         image: cr.l5d.io/linkerd/proxy:install-proxy-version
         imagePullPolicy: IfNotPresent
-        lifecycle:
-          postStart:
-            exec:
-              command:
-              - /usr/lib/linkerd/linkerd-await
-              - --timeout=2m
-              - --port=4191
         livenessProbe:
           httpGet:
             path: /live
@@ -183,63 +220,26 @@ spec:
             port: 4191
           initialDelaySeconds: 2
           timeoutSeconds: 1
+        restartPolicy: Always
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
           runAsNonRoot: true
           runAsUser: 2102
           seccompProfile:
             type: RuntimeDefault
+        startupProbe:
+          failureThreshold: 120
+          httpGet:
+            path: /ready
+            port: 4191
+          periodSeconds: 1
         terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - mountPath: /var/run/linkerd/identity/end-entity
           name: linkerd-identity-end-entity
         - mountPath: /var/run/secrets/tokens
           name: linkerd-identity-token
-      - image: nginx
-        name: nginx
-        ports:
-        - containerPort: 80
-          name: http
-      initContainers:
-      - args:
-        - --firewall-bin-path
-        - iptables-nft
-        - --firewall-save-bin-path
-        - iptables-nft-save
-        - --ipv6=false
-        - --incoming-proxy-port
-        - "4143"
-        - --outgoing-proxy-port
-        - "4140"
-        - --proxy-uid
-        - "2102"
-        - --inbound-ports-to-ignore
-        - 4190,4191,4567,4568
-        - --outbound-ports-to-ignore
-        - 4567,4568
-        command:
-        - /usr/lib/linkerd/linkerd2-proxy-init
-        image: cr.l5d.io/linkerd/proxy:install-proxy-version
-        imagePullPolicy: IfNotPresent
-        name: linkerd-init
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-          privileged: false
-          readOnlyRootFilesystem: true
-          runAsGroup: 65534
-          runAsNonRoot: true
-          runAsUser: 65534
-          seccompProfile:
-            type: RuntimeDefault
-        terminationMessagePolicy: FallbackToLogsOnError
-        volumeMounts:
-        - mountPath: /run
-          name: linkerd-proxy-init-xtables-lock
       volumes:
       - emptyDir: {}
         name: linkerd-proxy-init-xtables-lock