feat: add drain delay to graceful shutdown process

When a pod terminates, the shutdown-manager immediately calls
`healthcheck/fail` to fail health checks. This causes the readiness
probe to fail and removes the pod from service endpoints.

External load balancers that use health checks for deregistration need
several failed probes before they stop sending new connections to the
pod. On GKE, the L4 passthrough load balancer probes every 3 seconds
and needs 2 failures (at the time of writing), so deregistration takes 6 seconds or more.
The pod readiness probe fails faster than that, so for a few seconds the load
balancer keeps sending connections that kube-proxy no longer routes.

Kubernetes has a mechanism for this window. When a service has no
other ready endpoints, kube-proxy keeps routing to terminating pods
that still pass their readiness probe (KEP-1669). This only works if
the pod keeps passing its probe while the load balancer deregisters
it. Calling `healthcheck/fail` at the start of shutdown breaks that.

This adds a drainDelay field to ShutdownConfig which delays the
`healthcheck/fail` call. During the delay the pod stays ready and keeps
serving, so the load balancer has time to deregister it. The
ready-timeout and termination grace period are extended by the delay.

Signed-off-by: Michael Wain <michael@sanity.io>

Author: mwain

api/v1alpha1/envoyproxy_helpers.go

@@ -9,12 +9,17 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"time"
 
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
+// DefaultDrainTimeout is the default drain timeout for the graceful drain
+// sequence, used when ShutdownConfig.DrainTimeout is not specified.
+const DefaultDrainTimeout = 60 * time.Second
+
 // DefaultEnvoyProxyProvider returns a new EnvoyProxyProvider with default settings.
 func DefaultEnvoyProxyProvider() *EnvoyProxyProvider {
 	return &EnvoyProxyProvider{

internal/cmd/envoy.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/cmd/envoy"
 )
 
@@ -28,6 +29,7 @@ func GetEnvoyCommand() *cobra.Command {
 
 // getShutdownCommand returns the shutdown cobra command to be executed.
 func getShutdownCommand() *cobra.Command {
+	var drainDelay time.Duration
 	var drainTimeout time.Duration
 	var minDrainDuration time.Duration
 	var exitAtConnections int
@@ -36,11 +38,14 @@ func getShutdownCommand() *cobra.Command {
 		Use:   "shutdown",
 		Short: "Gracefully drain open connections prior to pod shutdown.",
 		RunE: func(_ *cobra.Command, _ []string) error {
-			return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections)
+			return envoy.Shutdown(drainDelay, drainTimeout, minDrainDuration, exitAtConnections)
 		},
 	}
 
-	cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second,
+	cmd.PersistentFlags().DurationVar(&drainDelay, "drain-delay", 0*time.Second,
+		"Delay before starting the drain process.")
+
+	cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", egv1a1.DefaultDrainTimeout,
 		"Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.")
 
 	cmd.PersistentFlags().DurationVar(&minDrainDuration, "min-drain-duration", 10*time.Second,

internal/cmd/envoy/shutdown_manager.go

@@ -118,15 +118,21 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea
 // Shutdown is called from a preStop hook on the shutdown-manager container where
 // it will initiate a drain sequence on the Envoy proxy and block until
 // connections are drained or a timeout is exceeded.
-func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
-	startTime := time.Now()
+func Shutdown(drainDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
 	allowedToExit := false
 
 	// Reconfigure logger to write to stdout of main process if running in Kubernetes
 	if _, k8s := os.LookupEnv("KUBERNETES_SERVICE_HOST"); k8s && os.Getpid() != 1 {
 		logger = logging.FileLogger("/proc/1/fd/1", "shutdown-manager", egv1a1.LogLevelInfo)
 	}
 
+	if drainDelay > 0 {
+		logger.Info(fmt.Sprintf("waiting %.0f seconds before starting drain", drainDelay.Seconds()))
+		time.Sleep(drainDelay)
+	}
+
+	startTime := time.Now()
+
 	logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout",
 		minDrainDuration.Seconds(), drainTimeout.Seconds()))
 

internal/infrastructure/common/proxy_args.go

@@ -84,7 +84,7 @@ func BuildProxyArgs(
 	}
 
 	// Default drain timeout.
-	drainTimeout := 60.0
+	drainTimeout := egv1a1.DefaultDrainTimeout.Seconds()
 	if shutdownConfig != nil && shutdownConfig.DrainTimeout != nil {
 		d, err := time.ParseDuration(string(*shutdownConfig.DrainTimeout))
 		if err != nil {

internal/infrastructure/kubernetes/proxy/resource.go

@@ -266,15 +266,32 @@ func expectedShutdownManagerImage(shutdownManager *egv1a1.ShutdownManager) strin
 }
 
 func expectedShutdownManagerArgs(cfg *egv1a1.ShutdownConfig) []string {
-	args := []string{"envoy", "shutdown-manager"}
-	if cfg != nil && cfg.DrainTimeout != nil {
+	args := make([]string, 0, 3)
+	args = append(args, "envoy", "shutdown-manager")
+
+	// Use default --ready-timeout when neither drain field is configured.
+	if cfg == nil || (cfg.DrainTimeout == nil && cfg.DrainDelay == nil) {
+		return args
+	}
+
+	readyTimeout := egv1a1.DefaultDrainTimeout
+	if cfg.DrainTimeout != nil {
 		d, err := time.ParseDuration(string(*cfg.DrainTimeout))
 		if err != nil {
 			return nil
 		}
-		args = append(args, fmt.Sprintf("--ready-timeout=%.0fs", d.Seconds()+10))
+		readyTimeout = d
 	}
-	return args
+
+	if cfg.DrainDelay != nil {
+		delay, err := time.ParseDuration(string(*cfg.DrainDelay))
+		if err != nil {
+			return nil
+		}
+		readyTimeout += delay
+	}
+
+	return append(args, fmt.Sprintf("--ready-timeout=%.0fs", readyTimeout.Seconds()+10))
 }
 
 func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
@@ -284,6 +301,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
 		return command
 	}
 
+	if cfg.DrainDelay != nil {
+		d, err := time.ParseDuration(string(*cfg.DrainDelay))
+		if err != nil {
+			return nil
+		}
+		command = append(command, fmt.Sprintf("--drain-delay=%.0fs", d.Seconds()))
+	}
+
 	if cfg.DrainTimeout != nil {
 		d, err := time.ParseDuration(string(*cfg.DrainTimeout))
 		if err != nil {

internal/infrastructure/kubernetes/proxy/resource_provider.go

@@ -620,6 +620,15 @@ func expectedTerminationGracePeriodSeconds(cfg *egv1a1.ShutdownConfig) *int64 {
 		}
 		s = int(d.Seconds() + 300) // 5 minutes longer than drain timeout
 	}
+
+	if cfg != nil && cfg.DrainDelay != nil {
+		d, err := time.ParseDuration(string(*cfg.DrainDelay))
+		if err != nil {
+			return nil
+		}
+		s += int(d.Seconds())
+	}
+
 	return new(int64(s))
 }
 

internal/infrastructure/kubernetes/proxy/resource_provider_test.go

@@ -292,6 +292,7 @@ func TestDeployment(t *testing.T) {
 				},
 			},
 			shutdown: &egv1a1.ShutdownConfig{
+				DrainDelay:       new(gwapiv1.Duration("15s")),
 				DrainTimeout:     new(gwapiv1.Duration("30s")),
 				MinDrainDuration: new(gwapiv1.Duration("15s")),
 			},
@@ -2120,3 +2121,52 @@ func writeTestDataToFile(filename string, resources []any) error {
 
 	return os.WriteFile(filename, combinedYAML, 0o600)
 }
+
+func TestExpectedTerminationGracePeriodSeconds(t *testing.T) {
+	tests := []struct {
+		name     string
+		cfg      *egv1a1.ShutdownConfig
+		expected int64
+	}{
+		{
+			name:     "nil config",
+			cfg:      nil,
+			expected: 360,
+		},
+		{
+			name:     "empty config",
+			cfg:      &egv1a1.ShutdownConfig{},
+			expected: 360,
+		},
+		{
+			name: "only drainTimeout",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainTimeout: new(gwapiv1.Duration("30s")),
+			},
+			expected: 330,
+		},
+		{
+			name: "only drainDelay",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainDelay: new(gwapiv1.Duration("15s")),
+			},
+			expected: 375,
+		},
+		{
+			name: "drainDelay and drainTimeout",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainDelay:   new(gwapiv1.Duration("15s")),
+				DrainTimeout: new(gwapiv1.Duration("30s")),
+			},
+			expected: 345,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := expectedTerminationGracePeriodSeconds(tt.cfg)
+			require.NotNil(t, got)
+			require.Equal(t, tt.expected, *got)
+		})
+	}
+}

internal/infrastructure/kubernetes/proxy/resource_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
@@ -182,3 +183,50 @@ func TestGetImageTag(t *testing.T) {
 		})
 	}
 }
+
+func TestExpectedShutdownManagerArgs(t *testing.T) {
+	tests := []struct {
+		name     string
+		cfg      *egv1a1.ShutdownConfig
+		expected []string
+	}{
+		{
+			name:     "nil config",
+			cfg:      nil,
+			expected: []string{"envoy", "shutdown-manager"},
+		},
+		{
+			name:     "empty config",
+			cfg:      &egv1a1.ShutdownConfig{},
+			expected: []string{"envoy", "shutdown-manager"},
+		},
+		{
+			name: "only drainTimeout",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainTimeout: new(gwapiv1.Duration("30s")),
+			},
+			expected: []string{"envoy", "shutdown-manager", "--ready-timeout=40s"},
+		},
+		{
+			name: "only drainDelay",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainDelay: new(gwapiv1.Duration("15s")),
+			},
+			expected: []string{"envoy", "shutdown-manager", "--ready-timeout=85s"},
+		},
+		{
+			name: "drainDelay and drainTimeout",
+			cfg: &egv1a1.ShutdownConfig{
+				DrainDelay:   new(gwapiv1.Duration("15s")),
+				DrainTimeout: new(gwapiv1.Duration("30s")),
+			},
+			expected: []string{"envoy", "shutdown-manager", "--ready-timeout=55s"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.expected, expectedShutdownManagerArgs(tt.cfg))
+		})
+	}
+}

internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml

@@ -309,7 +309,7 @@ spec:
       - args:
         - envoy
         - shutdown-manager
-        - --ready-timeout=40s
+        - --ready-timeout=55s
         command:
         - envoy-gateway
         env:
@@ -341,6 +341,7 @@ spec:
               - envoy-gateway
               - envoy
               - shutdown
+              - --drain-delay=15s
               - --drain-timeout=30s
               - --min-drain-duration=15s
         livenessProbe:
@@ -395,7 +396,7 @@ spec:
       restartPolicy: Always
       schedulerName: default-scheduler
       serviceAccountName: envoy-default-37a8eec1
-      terminationGracePeriodSeconds: 330
+      terminationGracePeriodSeconds: 345
       volumes:
       - name: certs
         secret:

release-notes/current.yaml

@@ -13,6 +13,7 @@ security updates: |
 new features: |
   Added support for authorization path match.
   Added a `requestBody` field to the HTTP active health checker in `BackendTrafficPolicy`, allowing a request body payload to be sent during HTTP health checking. The field requires the health check `method` to be `POST` or `PUT`.
+  Added a `drainDelay` field to `ShutdownConfig`, delaying the start of the Envoy proxy drain sequence on pod termination. During the delay the proxy continues to accept and serve new connections, allowing external load balancers that rely on health-check based de-registration (e.g. L4 passthrough load balancers) to stop sending traffic to the terminating pod before it stops serving.
 
 bug fixes: |
   Fixed Backend TLS `alpnProtocols: []` to disable upstream ALPN instead of inheriting EnvoyProxy BackendTLS defaults.