Skip to content

Commit f5c2235

Browse files
consideRatioconsideRatio
committed
feat: add healthCheckFailureDelay to shutdown config
Allow Envoy listener drain to start immediately while delaying `/healthcheck/fail` during pod termination. This helps deployments that need the terminating pod to remain a ready local endpoint while upstream load balancers stop sending traffic to the node. The default remains 0s, preserving the existing behavior where `/healthcheck/fail` starts listener drain immediately. Signed-off-by: Erik Sundell <erik@sundellopensource.se>
1 parent 8dcac65 commit f5c2235

14 files changed

Lines changed: 150 additions & 16 deletions

File tree

‎api/v1alpha1/envoyproxy_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,11 @@ type EnvoyProxyProvider struct {
508508

509509
// ShutdownConfig defines configuration for graceful envoy shutdown process.
510510
type ShutdownConfig struct {
511+
// HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
512+
// If unspecified, defaults to 0 seconds.
513+
//
514+
// +optional
515+
HealthCheckFailureDelay *gwapiv1.Duration `json:"healthCheckFailureDelay,omitempty"`
511516
// DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
512517
// If unspecified, defaults to 60 seconds.
513518
//

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11306,6 +11306,12 @@ spec:
1130611306
If unspecified, defaults to 60 seconds.
1130711307
pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
1130811308
type: string
11309+
healthCheckFailureDelay:
11310+
description: |-
11311+
HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
11312+
If unspecified, defaults to 0 seconds.
11313+
pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
11314+
type: string
1130911315
minDrainDuration:
1131011316
description: |-
1131111317
MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.

‎charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11305,6 +11305,12 @@ spec:
1130511305
If unspecified, defaults to 60 seconds.
1130611306
pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
1130711307
type: string
11308+
healthCheckFailureDelay:
11309+
description: |-
11310+
HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.
11311+
If unspecified, defaults to 0 seconds.
11312+
pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
11313+
type: string
1130811314
minDrainDuration:
1130911315
description: |-
1131011316
MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.

‎internal/cmd/envoy.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ func GetEnvoyCommand() *cobra.Command {
2828

2929
// getShutdownCommand returns the shutdown cobra command to be executed.
3030
func getShutdownCommand() *cobra.Command {
31+
var healthCheckFailureDelay time.Duration
3132
var drainTimeout time.Duration
3233
var minDrainDuration time.Duration
3334
var exitAtConnections int
@@ -36,10 +37,13 @@ func getShutdownCommand() *cobra.Command {
3637
Use: "shutdown",
3738
Short: "Gracefully drain open connections prior to pod shutdown.",
3839
RunE: func(_ *cobra.Command, _ []string) error {
39-
return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections)
40+
return envoy.Shutdown(healthCheckFailureDelay, drainTimeout, minDrainDuration, exitAtConnections)
4041
},
4142
}
4243

44+
cmd.PersistentFlags().DurationVar(&healthCheckFailureDelay, "health-check-failure-delay", 0*time.Second,
45+
"Delay before failing health checks during the graceful drain process.")
46+
4347
cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second,
4448
"Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.")
4549

‎internal/cmd/envoy/shutdown_manager.go‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,10 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea
118118
// Shutdown is called from a preStop hook on the shutdown-manager container where
119119
// it will initiate a drain sequence on the Envoy proxy and block until
120120
// connections are drained or a timeout is exceeded.
121-
func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
121+
func Shutdown(healthCheckFailureDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
122122
startTime := time.Now()
123123
allowedToExit := false
124+
healthCheckFailurePending := false
124125

125126
// Reconfigure logger to write to stdout of main process if running in Kubernetes
126127
if _, k8s := os.LookupEnv("KUBERNETES_SERVICE_HOST"); k8s && os.Getpid() != 1 {
@@ -130,16 +131,27 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
130131
logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout",
131132
minDrainDuration.Seconds(), drainTimeout.Seconds()))
132133

133-
// Start failing active health checks
134-
if err := postEnvoyAdminAPI("healthcheck/fail"); err != nil {
135-
logger.Error(err, "error failing active health checks")
134+
if healthCheckFailureDelay > 0 {
135+
if err := postEnvoyAdminAPI("drain_listeners?graceful&skip_exit"); err != nil {
136+
logger.Error(err, "error starting listener drain")
137+
}
138+
logger.Info(fmt.Sprintf("delaying health check failure by %.0f seconds", healthCheckFailureDelay.Seconds()))
139+
healthCheckFailurePending = true
140+
} else {
141+
// Failing active health checks also starts Envoy listener drain.
142+
failActiveHealthChecks(postEnvoyAdminAPI)
136143
}
137144

138145
// Poll total connections from Envoy admin API until minimum drain period has
139146
// been reached and total connections reaches threshold or timeout is exceeded
140147
for {
141148
elapsedTime := time.Since(startTime)
142149

150+
if healthCheckFailurePending && elapsedTime >= healthCheckFailureDelay {
151+
failActiveHealthChecks(postEnvoyAdminAPI)
152+
healthCheckFailurePending = false
153+
}
154+
143155
conn, err := getTotalConnections(bootstrap.EnvoyAdminPort)
144156
if err != nil {
145157
logger.Error(err, "error getting total connections")
@@ -170,6 +182,12 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
170182
return nil
171183
}
172184

185+
func failActiveHealthChecks(post func(string) error) {
186+
if err := post("healthcheck/fail"); err != nil {
187+
logger.Error(err, "error failing active health checks")
188+
}
189+
}
190+
173191
// postEnvoyAdminAPI sends a POST request to the Envoy admin API
174192
func postEnvoyAdminAPI(path string) error {
175193
resp, err := http.Post(fmt.Sprintf("http://%s:%d/%s",

‎internal/infrastructure/kubernetes/proxy/resource.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
284284
return command
285285
}
286286

287+
if cfg.HealthCheckFailureDelay != nil {
288+
d, err := time.ParseDuration(string(*cfg.HealthCheckFailureDelay))
289+
if err != nil {
290+
return nil
291+
}
292+
command = append(command, fmt.Sprintf("--health-check-failure-delay=%.0fs", d.Seconds()))
293+
}
294+
287295
if cfg.DrainTimeout != nil {
288296
d, err := time.ParseDuration(string(*cfg.DrainTimeout))
289297
if err != nil {

‎internal/infrastructure/kubernetes/proxy/resource_test.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"github.com/stretchr/testify/require"
1414
corev1 "k8s.io/api/core/v1"
15+
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
1516

1617
egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
1718
"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
@@ -182,3 +183,39 @@ func TestGetImageTag(t *testing.T) {
182183
})
183184
}
184185
}
186+
187+
func TestExpectedShutdownPreStopCommand(t *testing.T) {
188+
tests := []struct {
189+
name string
190+
cfg *egv1a1.ShutdownConfig
191+
expected []string
192+
}{
193+
{
194+
name: "nil config",
195+
cfg: nil,
196+
expected: []string{"envoy-gateway", "envoy", "shutdown"},
197+
},
198+
{
199+
name: "health check failure delay",
200+
cfg: &egv1a1.ShutdownConfig{
201+
HealthCheckFailureDelay: new(gwapiv1.Duration("15s")),
202+
DrainTimeout: new(gwapiv1.Duration("30s")),
203+
MinDrainDuration: new(gwapiv1.Duration("5s")),
204+
},
205+
expected: []string{
206+
"envoy-gateway",
207+
"envoy",
208+
"shutdown",
209+
"--health-check-failure-delay=15s",
210+
"--drain-timeout=30s",
211+
"--min-drain-duration=5s",
212+
},
213+
},
214+
}
215+
216+
for _, tt := range tests {
217+
t.Run(tt.name, func(t *testing.T) {
218+
require.Equal(t, tt.expected, expectedShutdownPreStopCommand(tt.cfg))
219+
})
220+
}
221+
}

‎release-notes/current/new_features/9210-healthcheckfailuredelay-shutdownconfig.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Added `healthCheckFailureDelay` to `ShutdownConfig`, allowing Envoy Gateway to start graceful listener drain immediately while delaying health check failure during pod termination.

‎site/content/en/latest/api/extension_types.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6051,6 +6051,7 @@ _Appears in:_
60516051

60526052
| Field | Type | Required | Default | Description |
60536053
| --- | --- | --- | --- | --- |
6054+
| `healthCheckFailureDelay` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | HealthCheckFailureDelay defines the delay before failing health checks during the graceful drain process.<br />If unspecified, defaults to 0 seconds. |
60546055
| `drainTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.<br />If unspecified, defaults to 60 seconds. |
60556056
| `minDrainDuration` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.<br />If unspecified, defaults to 10 seconds. |
60566057

0 commit comments

Comments
 (0)