diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 75226f6076..b7457a0a70 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -508,6 +508,11 @@ type EnvoyProxyProvider struct { // ShutdownConfig defines configuration for graceful envoy shutdown process. type ShutdownConfig struct { + // HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process. + // If unspecified, defaults to 0 seconds. + // + // +optional + HealthCheckFailureDelay *gwapiv1.Duration `json:"healthCheckFailureDelay,omitempty"` // DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds. // If unspecified, defaults to 60 seconds. // diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b966f945df..afc2650144 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8202,6 +8202,11 @@ func (in *SessionResumption) DeepCopy() *SessionResumption { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ShutdownConfig) DeepCopyInto(out *ShutdownConfig) { *out = *in + if in.HealthCheckFailureDelay != nil { + in, out := &in.HealthCheckFailureDelay, &out.HealthCheckFailureDelay + *out = new(v1.Duration) + **out = **in + } if in.DrainTimeout != nil { in, out := &in.DrainTimeout, &out.DrainTimeout *out = new(v1.Duration) diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml index 649f37b137..e323e39809 100644 --- a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -11306,6 +11306,12 @@ spec: If unspecified, defaults to 60 seconds. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + healthCheckFailureDelay: + description: |- + HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string minDrainDuration: description: |- MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete. diff --git a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index c213f6d051..d6337236e1 100644 --- a/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -11305,6 +11305,12 @@ spec: If unspecified, defaults to 60 seconds. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + healthCheckFailureDelay: + description: |- + HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string minDrainDuration: description: |- MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete. diff --git a/internal/cmd/envoy.go b/internal/cmd/envoy.go index 8c2d0f46ac..99c6f1ede4 100644 --- a/internal/cmd/envoy.go +++ b/internal/cmd/envoy.go @@ -28,6 +28,7 @@ func GetEnvoyCommand() *cobra.Command { // getShutdownCommand returns the shutdown cobra command to be executed. func getShutdownCommand() *cobra.Command { + var healthCheckFailureDelay time.Duration var drainTimeout time.Duration var minDrainDuration time.Duration var exitAtConnections int @@ -36,10 +37,13 @@ func getShutdownCommand() *cobra.Command { Use: "shutdown", Short: "Gracefully drain open connections prior to pod shutdown.", RunE: func(_ *cobra.Command, _ []string) error { - return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections) + return envoy.Shutdown(healthCheckFailureDelay, drainTimeout, minDrainDuration, exitAtConnections) }, } + cmd.PersistentFlags().DurationVar(&healthCheckFailureDelay, "health-check-failure-delay", 0*time.Second, + "Delay before failing health checks during the graceful drain process.") + cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second, "Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.") diff --git a/internal/cmd/envoy/shutdown_manager.go b/internal/cmd/envoy/shutdown_manager.go index cd56853392..6dfaa3da8e 100644 --- a/internal/cmd/envoy/shutdown_manager.go +++ b/internal/cmd/envoy/shutdown_manager.go @@ -118,9 +118,10 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea // Shutdown is called from a preStop hook on the shutdown-manager container where // it will initiate a drain sequence on the Envoy proxy and block until // connections are drained or a timeout is exceeded. -func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error { +func Shutdown(healthCheckFailureDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error { startTime := time.Now() allowedToExit := false + healthCheckFailurePending := false // Reconfigure logger to write to stdout of main process if running in Kubernetes if _, k8s := os.LookupEnv("KUBERNETES_SERVICE_HOST"); k8s && os.Getpid() != 1 { @@ -130,9 +131,15 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout", minDrainDuration.Seconds(), drainTimeout.Seconds())) - // Start failing active health checks - if err := postEnvoyAdminAPI("healthcheck/fail"); err != nil { - logger.Error(err, "error failing active health checks") + if healthCheckFailureDelay > 0 { + if err := postEnvoyAdminAPI("drain_listeners?graceful&skip_exit"); err != nil { + logger.Error(err, "error starting listener drain") + } + logger.Info(fmt.Sprintf("delaying health check failure by %.0f seconds", healthCheckFailureDelay.Seconds())) + healthCheckFailurePending = true + } else { + // Failing active health checks also starts Envoy listener drain. + failActiveHealthChecks(postEnvoyAdminAPI) } // Poll total connections from Envoy admin API until minimum drain period has @@ -140,6 +147,11 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in for { elapsedTime := time.Since(startTime) + if healthCheckFailurePending && elapsedTime >= healthCheckFailureDelay { + failActiveHealthChecks(postEnvoyAdminAPI) + healthCheckFailurePending = false + } + conn, err := getTotalConnections(bootstrap.EnvoyAdminPort) if err != nil { logger.Error(err, "error getting total connections") @@ -170,6 +182,12 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in return nil } +func failActiveHealthChecks(post func(string) error) { + if err := post("healthcheck/fail"); err != nil { + logger.Error(err, "error failing active health checks") + } +} + // postEnvoyAdminAPI sends a POST request to the Envoy admin API func postEnvoyAdminAPI(path string) error { resp, err := http.Post(fmt.Sprintf("http://%s:%d/%s", diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index 40a39d4e91..7b915e4cd7 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -284,6 +284,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string { return command } + if cfg.HealthCheckFailureDelay != nil { + d, err := time.ParseDuration(string(*cfg.HealthCheckFailureDelay)) + if err != nil { + return nil + } + command = append(command, fmt.Sprintf("--health-check-failure-delay=%s", d.String())) + } + if cfg.DrainTimeout != nil { d, err := time.ParseDuration(string(*cfg.DrainTimeout)) if err != nil { diff --git a/internal/infrastructure/kubernetes/proxy/resource_test.go b/internal/infrastructure/kubernetes/proxy/resource_test.go index 2428311495..aac7f97063 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" @@ -182,3 +183,51 @@ func TestGetImageTag(t *testing.T) { }) } } + +func TestExpectedShutdownPreStopCommand(t *testing.T) { + tests := []struct { + name string + cfg *egv1a1.ShutdownConfig + expected []string + }{ + { + name: "nil config", + cfg: nil, + expected: []string{"envoy-gateway", "envoy", "shutdown"}, + }, + { + name: "health check failure delay", + cfg: &egv1a1.ShutdownConfig{ + HealthCheckFailureDelay: new(gwapiv1.Duration("15s")), + DrainTimeout: new(gwapiv1.Duration("30s")), + MinDrainDuration: new(gwapiv1.Duration("5s")), + }, + expected: []string{ + "envoy-gateway", + "envoy", + "shutdown", + "--health-check-failure-delay=15s", + "--drain-timeout=30s", + "--min-drain-duration=5s", + }, + }, + { + name: "subsecond health check failure delay", + cfg: &egv1a1.ShutdownConfig{ + HealthCheckFailureDelay: new(gwapiv1.Duration("400ms")), + }, + expected: []string{ + "envoy-gateway", + "envoy", + "shutdown", + "--health-check-failure-delay=400ms", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, expectedShutdownPreStopCommand(tt.cfg)) + }) + } +} diff --git a/release-notes/current/new_features/9210-healthcheckfailuredelay-shutdownconfig.md b/release-notes/current/new_features/9210-healthcheckfailuredelay-shutdownconfig.md new file mode 100644 index 0000000000..a8b33a7753 --- /dev/null +++ b/release-notes/current/new_features/9210-healthcheckfailuredelay-shutdownconfig.md @@ -0,0 +1 @@ +Added `healthCheckFailureDelay` to `ShutdownConfig`, allowing Envoy Gateway to start graceful listener drain immediately while delaying health check failure during pod termination. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 12e60a35cc..703138bba7 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6051,6 +6051,7 @@ _Appears in:_ | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | +| `healthCheckFailureDelay` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
If unspecified, defaults to 0 seconds. | | `drainTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
If unspecified, defaults to 60 seconds. | | `minDrainDuration` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.
If unspecified, defaults to 10 seconds. | diff --git a/site/content/en/latest/tasks/operations/graceful-shutdown.md b/site/content/en/latest/tasks/operations/graceful-shutdown.md index 3f6bc08f20..7cc2d8f4a7 100644 --- a/site/content/en/latest/tasks/operations/graceful-shutdown.md +++ b/site/content/en/latest/tasks/operations/graceful-shutdown.md @@ -14,13 +14,20 @@ The shutdown manager sidecar coordinates graceful connection draining during pod ### Shutdown Process -1. Kubernetes sends SIGTERM to the pod -2. Shutdown manager fails health checks via `/healthcheck/fail` - - This causes Kubernetes readiness probes to fail - - External load balancers and services stop routing new traffic to the pod - - Existing connections continue to be served while draining -3. Connection monitoring begins, polling `server.total_connections` -4. Process exits when connections reach zero or drain timeout is exceeded +1. Kubernetes sends SIGTERM to the pod's containers and marks the pod as + terminating. +2. Shutdown manager starts Envoy listener drain. + - Drain is initiated directly via + `/drain_listeners?graceful&skip_exit` or indirectly via `/healthcheck/fail` + when no health check failure delay is configured. + - Envoy continues to serve accepted connections while listeners are draining. +3. Shutdown manager fails health checks via `/healthcheck/fail`, causing the + pod's readiness probe to fail. + - By default this happens immediately and also starts listener drain. + - When `healthCheckFailureDelay` is configured, this step is delayed without + delaying listener drain, connection monitoring, or the drain timeout. +4. Connection monitoring begins, polling `server.total_connections` +5. Process exits when connections reach zero or drain timeout is exceeded ## Configuration @@ -31,6 +38,13 @@ Graceful shutdown behavior includes default values that can be overridden using **Default Values:** - `drainTimeout`: 60 seconds - Maximum time for connection draining - `minDrainDuration`: 10 seconds - Minimum wait before allowing exit +- `healthCheckFailureDelay`: 0 seconds - Optional delay before failing health checks after drain starts + +`healthCheckFailureDelay` does not extend the drain sequence or keep the pod's +containers running. If the drain completes before `healthCheckFailureDelay` +elapses, `/healthcheck/fail` is not called. This can happen when connections +drop below `exitAtConnections` after `minDrainDuration`, or when +`healthCheckFailureDelay` is greater than or equal to `drainTimeout`. {{< tabpane text=true >}} {{% tab header="Gateway-Level Configuration" %}} @@ -58,8 +72,9 @@ metadata: name: graceful-shutdown-config spec: shutdown: - drainTimeout: "90s" # Override default 60s - minDrainDuration: "15s" # Override default 10s + drainTimeout: "90s" # Override default 60s + minDrainDuration: "15s" # Override default 10s + healthCheckFailureDelay: "40s" # Override default 0s ``` {{% /tab %}} @@ -83,8 +98,9 @@ metadata: name: graceful-shutdown-config spec: shutdown: - drainTimeout: "90s" # Override default 60s - minDrainDuration: "15s" # Override default 10s + drainTimeout: "90s" # Override default 60s + minDrainDuration: "15s" # Override default 10s + healthCheckFailureDelay: "40s" # Override default 0s ``` {{% /tab %}} diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml index 62f7f1e130..be16d11753 100644 --- a/test/helm/gateway-crds-helm/all.out.yaml +++ b/test/helm/gateway-crds-helm/all.out.yaml @@ -45134,6 +45134,12 @@ spec: If unspecified, defaults to 60 seconds. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + healthCheckFailureDelay: + description: |- + HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string minDrainDuration: description: |- MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete. diff --git a/test/helm/gateway-crds-helm/e2e.out.yaml b/test/helm/gateway-crds-helm/e2e.out.yaml index 0120595e5c..648e947c71 100644 --- a/test/helm/gateway-crds-helm/e2e.out.yaml +++ b/test/helm/gateway-crds-helm/e2e.out.yaml @@ -21072,6 +21072,12 @@ spec: If unspecified, defaults to 60 seconds. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + healthCheckFailureDelay: + description: |- + HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string minDrainDuration: description: |- MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete. diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml index babc7aa51c..c8c02c4565 100644 --- a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml +++ b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml @@ -21072,6 +21072,12 @@ spec: If unspecified, defaults to 60 seconds. pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ type: string + healthCheckFailureDelay: + description: |- + HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process. + If unspecified, defaults to 0 seconds. + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string minDrainDuration: description: |- MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.