feat: add healthCheckFailureDelay to shutdown config

api/v1alpha1/envoyproxy_types.go

@@ -508,6 +508,11 @@ type EnvoyProxyProvider struct {
 
 // ShutdownConfig defines configuration for graceful envoy shutdown process.
 type ShutdownConfig struct {
+	// HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+	// If unspecified, defaults to 0 seconds.
+	//
+	// +optional
+	HealthCheckFailureDelay *gwapiv1.Duration `json:"healthCheckFailureDelay,omitempty"`
 	// DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
 	// If unspecified, defaults to 60 seconds.
 	//

charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml

@@ -11306,6 +11306,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.

charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml

@@ -11305,6 +11305,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.

internal/cmd/envoy.go

@@ -28,6 +28,7 @@ func GetEnvoyCommand() *cobra.Command {
 
 // getShutdownCommand returns the shutdown cobra command to be executed.
 func getShutdownCommand() *cobra.Command {
+	var healthCheckFailureDelay time.Duration
 	var drainTimeout time.Duration
 	var minDrainDuration time.Duration
 	var exitAtConnections int
@@ -36,10 +37,13 @@ func getShutdownCommand() *cobra.Command {
 		Use:   "shutdown",
 		Short: "Gracefully drain open connections prior to pod shutdown.",
 		RunE: func(_ *cobra.Command, _ []string) error {
-			return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections)
+			return envoy.Shutdown(healthCheckFailureDelay, drainTimeout, minDrainDuration, exitAtConnections)
 		},
 	}
 
+	cmd.PersistentFlags().DurationVar(&healthCheckFailureDelay, "health-check-failure-delay", 0*time.Second,
+		"Delay before failing health checks during the graceful drain process.")
+
 	cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second,
 		"Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.")
 

internal/cmd/envoy/shutdown_manager.go

@@ -118,9 +118,10 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea
 // Shutdown is called from a preStop hook on the shutdown-manager container where
 // it will initiate a drain sequence on the Envoy proxy and block until
 // connections are drained or a timeout is exceeded.
-func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
+func Shutdown(healthCheckFailureDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
 	startTime := time.Now()
 	allowedToExit := false
+	healthCheckFailurePending := false
 
 	// Reconfigure logger to write to stdout of main process if running in Kubernetes
 	if _, k8s := os.LookupEnv("KUBERNETES_SERVICE_HOST"); k8s && os.Getpid() != 1 {
@@ -130,16 +131,27 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
 	logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout",
 		minDrainDuration.Seconds(), drainTimeout.Seconds()))
 
-	// Start failing active health checks
-	if err := postEnvoyAdminAPI("healthcheck/fail"); err != nil {
-		logger.Error(err, "error failing active health checks")
+	if healthCheckFailureDelay > 0 {
+		if err := postEnvoyAdminAPI("drain_listeners?graceful&skip_exit"); err != nil {
+			logger.Error(err, "error starting listener drain")
+		}
+		logger.Info(fmt.Sprintf("delaying health check failure by %.0f seconds", healthCheckFailureDelay.Seconds()))
+		healthCheckFailurePending = true
+	} else {
+		// Failing active health checks also starts Envoy listener drain.
+		failActiveHealthChecks(postEnvoyAdminAPI)
 	}
 
 	// Poll total connections from Envoy admin API until minimum drain period has
 	// been reached and total connections reaches threshold or timeout is exceeded
 	for {
 		elapsedTime := time.Since(startTime)
 
+		if healthCheckFailurePending && elapsedTime >= healthCheckFailureDelay {
+			failActiveHealthChecks(postEnvoyAdminAPI)
+			healthCheckFailurePending = false
+		}
+
 		conn, err := getTotalConnections(bootstrap.EnvoyAdminPort)
 		if err != nil {
 			logger.Error(err, "error getting total connections")
@@ -170,6 +182,12 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
 	return nil
 }
 
+func failActiveHealthChecks(post func(string) error) {
+	if err := post("healthcheck/fail"); err != nil {
+		logger.Error(err, "error failing active health checks")
+	}
+}
+
 // postEnvoyAdminAPI sends a POST request to the Envoy admin API
 func postEnvoyAdminAPI(path string) error {
 	resp, err := http.Post(fmt.Sprintf("http://%s:%d/%s",

internal/infrastructure/kubernetes/proxy/resource.go

@@ -284,6 +284,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
 		return command
 	}
 
+	if cfg.HealthCheckFailureDelay != nil {
+		d, err := time.ParseDuration(string(*cfg.HealthCheckFailureDelay))
+		if err != nil {
+			return nil
+		}
+		command = append(command, fmt.Sprintf("--health-check-failure-delay=%s", d.String()))
+	}
+
 	if cfg.DrainTimeout != nil {
 		d, err := time.ParseDuration(string(*cfg.DrainTimeout))
 		if err != nil {

internal/infrastructure/kubernetes/proxy/resource_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
@@ -182,3 +183,51 @@ func TestGetImageTag(t *testing.T) {
 		})
 	}
 }
+
+func TestExpectedShutdownPreStopCommand(t *testing.T) {
+	tests := []struct {
+		name     string
+		cfg      *egv1a1.ShutdownConfig
+		expected []string
+	}{
+		{
+			name:     "nil config",
+			cfg:      nil,
+			expected: []string{"envoy-gateway", "envoy", "shutdown"},
+		},
+		{
+			name: "health check failure delay",
+			cfg: &egv1a1.ShutdownConfig{
+				HealthCheckFailureDelay: new(gwapiv1.Duration("15s")),
+				DrainTimeout:            new(gwapiv1.Duration("30s")),
+				MinDrainDuration:        new(gwapiv1.Duration("5s")),
+			},
+			expected: []string{
+				"envoy-gateway",
+				"envoy",
+				"shutdown",
+				"--health-check-failure-delay=15s",
+				"--drain-timeout=30s",
+				"--min-drain-duration=5s",
+			},
+		},
+		{
+			name: "subsecond health check failure delay",
+			cfg: &egv1a1.ShutdownConfig{
+				HealthCheckFailureDelay: new(gwapiv1.Duration("400ms")),
+			},
+			expected: []string{
+				"envoy-gateway",
+				"envoy",
+				"shutdown",
+				"--health-check-failure-delay=400ms",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.expected, expectedShutdownPreStopCommand(tt.cfg))
+		})
+	}
+}

release-notes/current/new_features/9210-healthcheckfailuredelay-shutdownconfig.md

@@ -0,0 +1 @@
+Added `healthCheckFailureDelay` to `ShutdownConfig`, allowing Envoy Gateway to start graceful listener drain immediately while delaying health check failure during pod termination.

site/content/en/latest/api/extension_types.md

@@ -6051,6 +6051,7 @@ _Appears in:_
 
 | Field | Type | Required | Default | Description |
 | ---   | ---  | ---      | ---     | ---         |
+| `healthCheckFailureDelay` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.<br />If unspecified, defaults to 0 seconds. |
 | `drainTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.<br />If unspecified, defaults to 60 seconds. |
 | `minDrainDuration` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.<br />If unspecified, defaults to 10 seconds. |
 

site/content/en/latest/tasks/operations/graceful-shutdown.md

@@ -14,13 +14,20 @@ The shutdown manager sidecar coordinates graceful connection draining during pod
 
 ### Shutdown Process
 
-1. Kubernetes sends SIGTERM to the pod
-2. Shutdown manager fails health checks via `/healthcheck/fail`
-   - This causes Kubernetes readiness probes to fail
-   - External load balancers and services stop routing new traffic to the pod
-   - Existing connections continue to be served while draining
-3. Connection monitoring begins, polling `server.total_connections`
-4. Process exits when connections reach zero or drain timeout is exceeded
+1. Kubernetes sends SIGTERM to the pod's containers and marks the pod as
+   terminating.
+2. Shutdown manager starts Envoy listener drain.
+   - Drain is initiated directly via
+     `/drain_listeners?graceful&skip_exit` or indirectly via `/healthcheck/fail`
+     when no health check failure delay is configured.
+   - Envoy continues to serve accepted connections while listeners are draining.
+3. Shutdown manager fails health checks via `/healthcheck/fail`, causing the
+   pod's readiness probe to fail.
+   - By default this happens immediately and also starts listener drain.
+   - When `healthCheckFailureDelay` is configured, this step is delayed without
+     delaying listener drain, connection monitoring, or the drain timeout.
+4. Connection monitoring begins, polling `server.total_connections`
+5. Process exits when connections reach zero or drain timeout is exceeded
 
 ## Configuration
 
@@ -31,6 +38,13 @@ Graceful shutdown behavior includes default values that can be overridden using
 **Default Values:**
 - `drainTimeout`: 60 seconds - Maximum time for connection draining
 - `minDrainDuration`: 10 seconds - Minimum wait before allowing exit
+- `healthCheckFailureDelay`: 0 seconds - Optional delay before failing health checks after drain starts
+
+`healthCheckFailureDelay` does not extend the drain sequence or keep the pod's
+containers running. If the drain completes before `healthCheckFailureDelay`
+elapses, `/healthcheck/fail` is not called. This can happen when connections
+drop below `exitAtConnections` after `minDrainDuration`, or when
+`healthCheckFailureDelay` is greater than or equal to `drainTimeout`.
 
 {{< tabpane text=true >}}
 {{% tab header="Gateway-Level Configuration" %}}
@@ -58,8 +72,9 @@ metadata:
   name: graceful-shutdown-config
 spec:
   shutdown:
-    drainTimeout: "90s"      # Override default 60s
-    minDrainDuration: "15s"  # Override default 10s
+    drainTimeout: "90s"              # Override default 60s
+    minDrainDuration: "15s"          # Override default 10s
+    healthCheckFailureDelay: "40s"   # Override default 0s
 ```
 
 {{% /tab %}}
@@ -83,8 +98,9 @@ metadata:
   name: graceful-shutdown-config
 spec:
   shutdown:
-    drainTimeout: "90s"      # Override default 60s
-    minDrainDuration: "15s"  # Override default 10s
+    drainTimeout: "90s"              # Override default 60s
+    minDrainDuration: "15s"          # Override default 10s
+    healthCheckFailureDelay: "40s"   # Override default 0s
 ```
 
 {{% /tab %}}

test/helm/gateway-crds-helm/all.out.yaml

@@ -45134,6 +45134,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.

test/helm/gateway-crds-helm/e2e.out.yaml

@@ -21072,6 +21072,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.

test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml

@@ -21072,6 +21072,12 @@ spec:
                       If unspecified, defaults to 60 seconds.
                     pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
                     type: string
+                  healthCheckFailureDelay:
+                    description: |-
+                      HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
+                      If unspecified, defaults to 0 seconds.
+                    pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
+                    type: string
                   minDrainDuration:
                     description: |-
                       MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.