feat: add healthCheckFailureDelay to shutdown config

consideRatio · claude · consideRatio · commit e6be2a9ecfc3 · 2026-08-02T22:39:04.000+02:00
Allow Envoy listener drain to start immediately while delaying
`/healthcheck/fail` during pod termination. This helps deployments that
need the terminating pod to remain a ready local endpoint while
upstream load balancers stop sending traffic to the node.

The default remains 0s, preserving the existing behavior where
`/healthcheck/fail` starts listener drain immediately.

Signed-off-by: Erik Sundell &lt;erik@sundellopensource.se&gt;
Co-Authored-By: Claude Fable 5 &lt;noreply@anthropic.com&gt;
diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go
@@ -508,6 +508,11 @@ type EnvoyProxyProvider struct {
 
 // ShutdownConfig defines configuration for graceful envoy shutdown process.
 type ShutdownConfig struct {
+	// HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+	// If unspecified, defaults to 0 seconds.
+	//
+	// +optional
+	HealthCheckFailureDelay *gwapiv1.Duration `json:"healthCheckFailureDelay,omitempty"`
 	// DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
 	// If unspecified, defaults to 60 seconds.
 	//
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml
@@ -11306,6 +11306,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.
diff --git a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml
@@ -11305,6 +11305,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.
diff --git a/internal/cmd/envoy.go b/internal/cmd/envoy.go
@@ -28,6 +28,7 @@ func GetEnvoyCommand() *cobra.Command {
 
 // getShutdownCommand returns the shutdown cobra command to be executed.
 func getShutdownCommand() *cobra.Command {
+	var healthCheckFailureDelay time.Duration
 	var drainTimeout time.Duration
 	var minDrainDuration time.Duration
 	var exitAtConnections int
@@ -36,10 +37,13 @@ func getShutdownCommand() *cobra.Command {
 		Use:   "shutdown",
 		Short: "Gracefully drain open connections prior to pod shutdown.",
 		RunE: func(_ *cobra.Command, _ []string) error {
-			return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections)
+			return envoy.Shutdown(healthCheckFailureDelay, drainTimeout, minDrainDuration, exitAtConnections)
 		},
 	}
 
+	cmd.PersistentFlags().DurationVar(&healthCheckFailureDelay, "health-check-failure-delay", 0*time.Second,
+		"Delay before failing health checks during the graceful drain process.")
+
 	cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second,
 		"Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.")
 
diff --git a/internal/cmd/envoy/shutdown_manager.go b/internal/cmd/envoy/shutdown_manager.go
@@ -118,9 +118,10 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea
 // Shutdown is called from a preStop hook on the shutdown-manager container where
 // it will initiate a drain sequence on the Envoy proxy and block until
 // connections are drained or a timeout is exceeded.
-func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
+func Shutdown(healthCheckFailureDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
 	startTime := time.Now()
 	allowedToExit := false
+	healthCheckFailurePending := false
 
 	// Reconfigure logger to write to stdout of main process if running in Kubernetes
 	if _, k8s := os.LookupEnv("KUBERNETES_SERVICE_HOST"); k8s && os.Getpid() != 1 {
@@ -130,16 +131,27 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
 	logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout",
 		minDrainDuration.Seconds(), drainTimeout.Seconds()))
 
-	// Start failing active health checks
-	if err := postEnvoyAdminAPI("healthcheck/fail"); err != nil {
-		logger.Error(err, "error failing active health checks")
+	if healthCheckFailureDelay > 0 {
+		if err := postEnvoyAdminAPI("drain_listeners?graceful&skip_exit"); err != nil {
+			logger.Error(err, "error starting listener drain")
+		}
+		logger.Info(fmt.Sprintf("delaying health check failure by %.0f seconds", healthCheckFailureDelay.Seconds()))
+		healthCheckFailurePending = true
+	} else {
+		// Failing active health checks also starts Envoy listener drain.
+		failActiveHealthChecks(postEnvoyAdminAPI)
 	}
 
 	// Poll total connections from Envoy admin API until minimum drain period has
 	// been reached and total connections reaches threshold or timeout is exceeded
 	for {
 		elapsedTime := time.Since(startTime)
 
+		if healthCheckFailurePending && elapsedTime >= healthCheckFailureDelay {
+			failActiveHealthChecks(postEnvoyAdminAPI)
+			healthCheckFailurePending = false
+		}
+
 		conn, err := getTotalConnections(bootstrap.EnvoyAdminPort)
 		if err != nil {
 			logger.Error(err, "error getting total connections")
@@ -170,6 +182,12 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
 	return nil
 }
 
+func failActiveHealthChecks(post func(string) error) {
+	if err := post("healthcheck/fail"); err != nil {
+		logger.Error(err, "error failing active health checks")
+	}
+}
+
 // postEnvoyAdminAPI sends a POST request to the Envoy admin API
 func postEnvoyAdminAPI(path string) error {
 	resp, err := http.Post(fmt.Sprintf("http://%s:%d/%s",
diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go
@@ -284,6 +284,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
 		return command
 	}
 
+	if cfg.HealthCheckFailureDelay != nil {
+		d, err := time.ParseDuration(string(*cfg.HealthCheckFailureDelay))
+		if err != nil {
+			return nil
+		}
+		command = append(command, fmt.Sprintf("--health-check-failure-delay=%.0fs", d.Seconds()))
+	}
+
 	if cfg.DrainTimeout != nil {
 		d, err := time.ParseDuration(string(*cfg.DrainTimeout))
 		if err != nil {
diff --git a/internal/infrastructure/kubernetes/proxy/resource_test.go b/internal/infrastructure/kubernetes/proxy/resource_test.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
@@ -182,3 +183,39 @@ func TestGetImageTag(t *testing.T) {
 		})
 	}
 }
+
+func TestExpectedShutdownPreStopCommand(t *testing.T) {
+	tests := []struct {
+		name     string
+		cfg      *egv1a1.ShutdownConfig
+		expected []string
+	}{
+		{
+			name:     "nil config",
+			cfg:      nil,
+			expected: []string{"envoy-gateway", "envoy", "shutdown"},
+		},
+		{
+			name: "health check failure delay",
+			cfg: &egv1a1.ShutdownConfig{
+				HealthCheckFailureDelay: new(gwapiv1.Duration("15s")),
+				DrainTimeout:          new(gwapiv1.Duration("30s")),
+				MinDrainDuration:      new(gwapiv1.Duration("5s")),
+			},
+			expected: []string{
+				"envoy-gateway",
+				"envoy",
+				"shutdown",
+				"--health-check-failure-delay=15s",
+				"--drain-timeout=30s",
+				"--min-drain-duration=5s",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.expected, expectedShutdownPreStopCommand(tt.cfg))
+		})
+	}
+}
diff --git a/release-notes/current/new_features/9210-healthcheckfailuredelay-shutdownconfig.md b/release-notes/current/new_features/9210-healthcheckfailuredelay-shutdownconfig.md
@@ -0,0 +1 @@
+Added `healthCheckFailureDelay` to `ShutdownConfig`, allowing Envoy Gateway to start graceful listener drain immediately while delaying health check failure during pod termination.
diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md
@@ -6051,6 +6051,7 @@ _Appears in:_
 
 | Field | Type | Required | Default | Description |
 | ---   | ---  | ---      | ---     | ---         |
+| `healthCheckFailureDelay` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.<br />If unspecified, defaults to 0 seconds. |
 | `drainTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.<br />If unspecified, defaults to 60 seconds. |
 | `minDrainDuration` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.<br />If unspecified, defaults to 10 seconds. |
 
diff --git a/site/content/en/latest/tasks/operations/graceful-shutdown.md b/site/content/en/latest/tasks/operations/graceful-shutdown.md
@@ -14,13 +14,20 @@ The shutdown manager sidecar coordinates graceful connection draining during pod
 
 ### Shutdown Process
 
-1. Kubernetes sends SIGTERM to the pod
-2. Shutdown manager fails health checks via `/healthcheck/fail`
-   - This causes Kubernetes readiness probes to fail
-   - External load balancers and services stop routing new traffic to the pod
-   - Existing connections continue to be served while draining
-3. Connection monitoring begins, polling `server.total_connections`
-4. Process exits when connections reach zero or drain timeout is exceeded
+1. Kubernetes sends SIGTERM to the pod's containers and marks the pod as
+   terminating.
+2. Shutdown manager starts Envoy listener drain.
+   - Drain is initiated directly via
+     `/drain_listeners?graceful&skip_exit` or indirectly via `/healthcheck/fail`
+     when no health check failure delay is configured.
+   - Envoy continues to serve accepted connections while listeners are draining.
+3. Shutdown manager fails health checks via `/healthcheck/fail`, causing the
+   pod's readiness probe to fail.
+   - By default this happens immediately and also starts listener drain.
+   - When `healthCheckFailureDelay` is configured, this step is delayed without
+     delaying listener drain, connection monitoring, or the drain timeout.
+4. Connection monitoring begins, polling `server.total_connections`
+5. Process exits when connections reach zero or drain timeout is exceeded
 
 ## Configuration
 
@@ -31,6 +38,21 @@ Graceful shutdown behavior includes default values that can be overridden using
 **Default Values:**
 - `drainTimeout`: 60 seconds - Maximum time for connection draining
 - `minDrainDuration`: 10 seconds - Minimum wait before allowing exit
+- `healthCheckFailureDelay`: 0 seconds - Optional delay before failing health checks after drain starts
+
+Configure `healthCheckFailureDelay` when failing health checks (and thereby
+readiness) immediately would leave the node without ready local endpoints
+before upstream load balancers have stopped sending traffic to it. This can be
+important when using `externalTrafficPolicy: Local` on the Kubernetes Service,
+where a node without ready local endpoints can stop receiving traffic even
+though the terminating Envoy pod is still able to serve connections during its
+drain period.
+
+`healthCheckFailureDelay` does not extend the drain sequence or keep the pod's
+containers running. If the drain completes before `healthCheckFailureDelay`
+elapses, `/healthcheck/fail` is not called. This can happen when connections
+drop below `exitAtConnections` after `minDrainDuration`, or when
+`healthCheckFailureDelay` is greater than or equal to `drainTimeout`.
 
 {{< tabpane text=true >}}
 {{% tab header="Gateway-Level Configuration" %}}
@@ -58,8 +80,9 @@ metadata:
   name: graceful-shutdown-config
 spec:
   shutdown:
-    drainTimeout: "90s"      # Override default 60s
-    minDrainDuration: "15s"  # Override default 10s
+    drainTimeout: "90s"              # Override default 60s
+    minDrainDuration: "15s"          # Override default 10s
+    healthCheckFailureDelay: "40s"   # Override default 0s
 ```
 
 {{% /tab %}}
@@ -83,8 +106,9 @@ metadata:
   name: graceful-shutdown-config
 spec:
   shutdown:
-    drainTimeout: "90s"      # Override default 60s
-    minDrainDuration: "15s"  # Override default 10s
+    drainTimeout: "90s"              # Override default 60s
+    minDrainDuration: "15s"          # Override default 10s
+    healthCheckFailureDelay: "40s"   # Override default 0s
 ```
 
 {{% /tab %}}
diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml
@@ -45134,6 +45134,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.
diff --git a/test/helm/gateway-crds-helm/e2e.out.yaml b/test/helm/gateway-crds-helm/e2e.out.yaml
@@ -21072,6 +21072,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.
diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml
@@ -21072,6 +21072,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Added `healthCheckFailureDelay` to `ShutdownConfig`, allowing Envoy Gateway to start graceful listener drain immediately while delaying health check failure during pod termination.