Skip to content

Commit 77d11ae

Browse files
consideRatioconsideRatio
committed
feat: add readinessFailureDelay to shutdown config
Allow Envoy listener drain to start immediately while delaying `/healthcheck/fail` during pod termination. This helps deployments that need the terminating pod to remain a ready local endpoint while upstream load balancers stop sending traffic to the node. The default remains 0s, preserving the existing behavior where `/healthcheck/fail` starts listener drain immediately. Signed-off-by: Erik Sundell <erik@sundellopensource.se>
1 parent 39850aa commit 77d11ae

14 files changed

Lines changed: 147 additions & 16 deletions

File tree

‎api/v1alpha1/envoyproxy_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,11 @@ type EnvoyProxyProvider struct {
489489

490490
// ShutdownConfig defines configuration for graceful envoy shutdown process.
491491
type ShutdownConfig struct {
492+
// ReadinessFailureDelay defines the delay before failing readiness during the graceful drain process.
493+
// If unspecified, defaults to 0 seconds.
494+
//
495+
// +optional
496+
ReadinessFailureDelay *gwapiv1.Duration `json:"readinessFailureDelay,omitempty"`
492497
// DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
493498
// If unspecified, defaults to 60 seconds.
494499
//

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11295,6 +11295,12 @@ spec:
1129511295
If unspecified, defaults to 10 seconds.
1129611296
pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
1129711297
type: string
11298+
readinessFailureDelay:
11299+
description: |-
11300+
ReadinessFailureDelay defines the delay before failing readiness during the graceful drain process.
11301+
If unspecified, defaults to 0 seconds.
11302+
pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
11303+
type: string
1129811304
type: object
1129911305
telemetry:
1130011306
description: Telemetry defines telemetry parameters for managed proxies.

‎charts/gateway-helm/charts/crds/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11294,6 +11294,12 @@ spec:
1129411294
If unspecified, defaults to 10 seconds.
1129511295
pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
1129611296
type: string
11297+
readinessFailureDelay:
11298+
description: |-
11299+
ReadinessFailureDelay defines the delay before failing readiness during the graceful drain process.
11300+
If unspecified, defaults to 0 seconds.
11301+
pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$
11302+
type: string
1129711303
type: object
1129811304
telemetry:
1129911305
description: Telemetry defines telemetry parameters for managed proxies.

‎internal/cmd/envoy.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ func GetEnvoyCommand() *cobra.Command {
2828

2929
// getShutdownCommand returns the shutdown cobra command to be executed.
3030
func getShutdownCommand() *cobra.Command {
31+
var readinessFailureDelay time.Duration
3132
var drainTimeout time.Duration
3233
var minDrainDuration time.Duration
3334
var exitAtConnections int
@@ -36,10 +37,13 @@ func getShutdownCommand() *cobra.Command {
3637
Use: "shutdown",
3738
Short: "Gracefully drain open connections prior to pod shutdown.",
3839
RunE: func(_ *cobra.Command, _ []string) error {
39-
return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections)
40+
return envoy.Shutdown(readinessFailureDelay, drainTimeout, minDrainDuration, exitAtConnections)
4041
},
4142
}
4243

44+
cmd.PersistentFlags().DurationVar(&readinessFailureDelay, "readiness-failure-delay", 0*time.Second,
45+
"Delay before failing readiness during the graceful drain process.")
46+
4347
cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 60*time.Second,
4448
"Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.")
4549

‎internal/cmd/envoy/shutdown_manager.go‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,10 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea
118118
// Shutdown is called from a preStop hook on the shutdown-manager container where
119119
// it will initiate a drain sequence on the Envoy proxy and block until
120120
// connections are drained or a timeout is exceeded.
121-
func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
121+
func Shutdown(readinessFailureDelay, drainTimeout, minDrainDuration time.Duration, exitAtConnections int) error {
122122
startTime := time.Now()
123123
allowedToExit := false
124+
readinessFailurePending := false
124125

125126
// Reconfigure logger to write to stdout of main process if running in Kubernetes
126127
if _, k8s := os.LookupEnv("KUBERNETES_SERVICE_HOST"); k8s && os.Getpid() != 1 {
@@ -130,16 +131,27 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
130131
logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout",
131132
minDrainDuration.Seconds(), drainTimeout.Seconds()))
132133

133-
// Start failing active health checks
134-
if err := postEnvoyAdminAPI("healthcheck/fail"); err != nil {
135-
logger.Error(err, "error failing active health checks")
134+
if readinessFailureDelay > 0 {
135+
if err := postEnvoyAdminAPI("drain_listeners?graceful&skip_exit"); err != nil {
136+
logger.Error(err, "error starting listener drain")
137+
}
138+
logger.Info(fmt.Sprintf("delaying readiness failure by %.0f seconds", readinessFailureDelay.Seconds()))
139+
readinessFailurePending = true
140+
} else {
141+
// Failing active health checks also starts Envoy listener drain.
142+
failActiveHealthChecks(postEnvoyAdminAPI)
136143
}
137144

138145
// Poll total connections from Envoy admin API until minimum drain period has
139146
// been reached and total connections reaches threshold or timeout is exceeded
140147
for {
141148
elapsedTime := time.Since(startTime)
142149

150+
if readinessFailurePending && elapsedTime >= readinessFailureDelay {
151+
failActiveHealthChecks(postEnvoyAdminAPI)
152+
readinessFailurePending = false
153+
}
154+
143155
conn, err := getTotalConnections(bootstrap.EnvoyAdminPort)
144156
if err != nil {
145157
logger.Error(err, "error getting total connections")
@@ -170,6 +182,12 @@ func Shutdown(drainTimeout, minDrainDuration time.Duration, exitAtConnections in
170182
return nil
171183
}
172184

185+
func failActiveHealthChecks(post func(string) error) {
186+
if err := post("healthcheck/fail"); err != nil {
187+
logger.Error(err, "error failing active health checks")
188+
}
189+
}
190+
173191
// postEnvoyAdminAPI sends a POST request to the Envoy admin API
174192
func postEnvoyAdminAPI(path string) error {
175193
resp, err := http.Post(fmt.Sprintf("http://%s:%d/%s",

‎internal/infrastructure/kubernetes/proxy/resource.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,14 @@ func expectedShutdownPreStopCommand(cfg *egv1a1.ShutdownConfig) []string {
284284
return command
285285
}
286286

287+
if cfg.ReadinessFailureDelay != nil {
288+
d, err := time.ParseDuration(string(*cfg.ReadinessFailureDelay))
289+
if err != nil {
290+
return nil
291+
}
292+
command = append(command, fmt.Sprintf("--readiness-failure-delay=%.0fs", d.Seconds()))
293+
}
294+
287295
if cfg.DrainTimeout != nil {
288296
d, err := time.ParseDuration(string(*cfg.DrainTimeout))
289297
if err != nil {

‎internal/infrastructure/kubernetes/proxy/resource_test.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"github.com/stretchr/testify/require"
1414
corev1 "k8s.io/api/core/v1"
15+
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
1516

1617
egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
1718
"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
@@ -182,3 +183,39 @@ func TestGetImageTag(t *testing.T) {
182183
})
183184
}
184185
}
186+
187+
func TestExpectedShutdownPreStopCommand(t *testing.T) {
188+
tests := []struct {
189+
name string
190+
cfg *egv1a1.ShutdownConfig
191+
expected []string
192+
}{
193+
{
194+
name: "nil config",
195+
cfg: nil,
196+
expected: []string{"envoy-gateway", "envoy", "shutdown"},
197+
},
198+
{
199+
name: "readiness failure delay",
200+
cfg: &egv1a1.ShutdownConfig{
201+
ReadinessFailureDelay: new(gwapiv1.Duration("15s")),
202+
DrainTimeout: new(gwapiv1.Duration("30s")),
203+
MinDrainDuration: new(gwapiv1.Duration("5s")),
204+
},
205+
expected: []string{
206+
"envoy-gateway",
207+
"envoy",
208+
"shutdown",
209+
"--readiness-failure-delay=15s",
210+
"--drain-timeout=30s",
211+
"--min-drain-duration=5s",
212+
},
213+
},
214+
}
215+
216+
for _, tt := range tests {
217+
t.Run(tt.name, func(t *testing.T) {
218+
require.Equal(t, tt.expected, expectedShutdownPreStopCommand(tt.cfg))
219+
})
220+
}
221+
}

‎release-notes/current/new_features/9210-readinessfailuredelay-shutdownconfig.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Added `readinessFailureDelay` to `ShutdownConfig`, allowing Envoy Gateway to start graceful listener drain immediately while delaying readiness failure during pod termination.

‎site/content/en/latest/api/extension_types.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5867,6 +5867,7 @@ _Appears in:_
58675867

58685868
| Field | Type | Required | Default | Description |
58695869
| --- | --- | --- | --- | --- |
5870+
| `readinessFailureDelay` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | ReadinessFailureDelay defines the delay before failing readiness during the graceful drain process.<br />If unspecified, defaults to 0 seconds. |
58705871
| `drainTimeout` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.<br />If unspecified, defaults to 60 seconds. |
58715872
| `minDrainDuration` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.<br />If unspecified, defaults to 10 seconds. |
58725873

0 commit comments

Comments
 (0)