Skip to content

Commit 1aaec4b

Browse files
chenggu88chenggu88
authored
Add shutdown_grace_period_seconds and shutdown_grace_period_critical_pods_seconds fields to GKE cluster kubelet_config (#17999)
1 parent 46c7041 commit 1aaec4b

8 files changed

Lines changed: 308 additions & 22 deletions

‎mmv1/third_party/terraform/services/container/node_config.go.tmpl‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -912,6 +912,18 @@ func schemaNodeConfig() *schema.Schema {
912912
Optional: true,
913913
Description: `Defines the maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met.`,
914914
},
915+
"shutdown_grace_period_seconds": {
916+
Type: schema.TypeInt,
917+
Optional: true,
918+
Computed: true,
919+
Description: `Controls the total duration of time (in seconds) the node delays shutdown.`,
920+
},
921+
"shutdown_grace_period_critical_pods_seconds": {
922+
Type: schema.TypeInt,
923+
Optional: true,
924+
Computed: true,
925+
Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`,
926+
},
915927
"eviction_soft": {
916928
Type: schema.TypeList,
917929
Optional: true,
@@ -1876,6 +1888,21 @@ func expandNodeConfig(d *schema.ResourceData, prefix string, v interface{}) *con
18761888
}
18771889
}
18781890
// end cpu_cfs_quota fix
1891+
1892+
// start shutdown_grace_period ForceSendFields fix
1893+
if vNC := rawConfigNPRoot.GetAttr("node_config"); vNC.LengthInt() > 0 {
1894+
if vKC := vNC.Index(cty.NumberIntVal(0)).GetAttr("kubelet_config"); vKC.LengthInt() > 0 {
1895+
vSGP := vKC.Index(cty.NumberIntVal(0)).GetAttr("shutdown_grace_period_seconds")
1896+
if vSGP != cty.NullVal(cty.Number) && !vSGP.IsNull() {
1897+
nc.KubeletConfig.ForceSendFields = append(nc.KubeletConfig.ForceSendFields, "ShutdownGracePeriodSeconds")
1898+
}
1899+
vSGPC := vKC.Index(cty.NumberIntVal(0)).GetAttr("shutdown_grace_period_critical_pods_seconds")
1900+
if vSGPC != cty.NullVal(cty.Number) && !vSGPC.IsNull() {
1901+
nc.KubeletConfig.ForceSendFields = append(nc.KubeletConfig.ForceSendFields, "ShutdownGracePeriodCriticalPodsSeconds")
1902+
}
1903+
}
1904+
}
1905+
// end shutdown_grace_period ForceSendFields fix
18791906
}
18801907

18811908
if v, ok := nodeConfig["linux_node_config"]; ok {
@@ -2080,6 +2107,12 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig {
20802107
if evictionMaxPodGracePeriodSeconds, ok := cfg["eviction_max_pod_grace_period_seconds"]; ok {
20812108
kConfig.EvictionMaxPodGracePeriodSeconds = int64(evictionMaxPodGracePeriodSeconds.(int))
20822109
}
2110+
if shutdownGracePeriodSeconds, ok := cfg["shutdown_grace_period_seconds"]; ok {
2111+
kConfig.ShutdownGracePeriodSeconds = int64(shutdownGracePeriodSeconds.(int))
2112+
}
2113+
if shutdownGracePeriodCriticalPodsSeconds, ok := cfg["shutdown_grace_period_critical_pods_seconds"]; ok {
2114+
kConfig.ShutdownGracePeriodCriticalPodsSeconds = int64(shutdownGracePeriodCriticalPodsSeconds.(int))
2115+
}
20832116
if v, ok := cfg["eviction_soft"]; ok && len(v.([]interface{})) > 0 {
20842117
es := v.([]interface{})[0].(map[string]interface{})
20852118
evictionSoft := &container.EvictionSignals{}
@@ -3324,6 +3357,8 @@ func flattenKubeletConfig(c *container.NodeKubeletConfig) []map[string]interface
33243357
"single_process_oom_kill": c.SingleProcessOomKill,
33253358
"max_parallel_image_pulls": c.MaxParallelImagePulls,
33263359
"eviction_max_pod_grace_period_seconds": c.EvictionMaxPodGracePeriodSeconds,
3360+
"shutdown_grace_period_seconds": c.ShutdownGracePeriodSeconds,
3361+
"shutdown_grace_period_critical_pods_seconds": c.ShutdownGracePeriodCriticalPodsSeconds,
33273362
"eviction_soft": flattenEvictionSignals(c.EvictionSoft),
33283363
"eviction_soft_grace_period": flattenEvictionGracePeriod(c.EvictionSoftGracePeriod),
33293364
"eviction_minimum_reclaim": flattenEvictionMinimumReclaim(c.EvictionMinimumReclaim),

‎mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,8 @@ fields:
359359
- api_field: 'nodeConfig.kubeletConfig.imageMaximumGcAge'
360360
- api_field: 'nodeConfig.kubeletConfig.imageMinimumGcAge'
361361
- api_field: 'nodeConfig.kubeletConfig.insecureKubeletReadonlyPortEnabled'
362+
- api_field: 'nodeConfig.kubeletConfig.shutdownGracePeriodSeconds'
363+
- api_field: 'nodeConfig.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
362364
- api_field: 'nodeConfig.kubeletConfig.maxParallelImagePulls'
363365
- api_field: 'nodeConfig.kubeletConfig.memoryManager.policy'
364366
- api_field: 'nodeConfig.kubeletConfig.podPidsLimit'
@@ -642,6 +644,10 @@ fields:
642644
api_field: 'nodePools.config.kubeletConfig.imageMinimumGcAge'
643645
- field: 'node_pool.node_config.kubelet_config.insecure_kubelet_readonly_port_enabled'
644646
api_field: 'nodePools.config.kubeletConfig.insecureKubeletReadonlyPortEnabled'
647+
- field: 'node_pool.node_config.kubelet_config.shutdown_grace_period_seconds'
648+
api_field: 'nodePools.config.kubeletConfig.shutdownGracePeriodSeconds'
649+
- field: 'node_pool.node_config.kubelet_config.shutdown_grace_period_critical_pods_seconds'
650+
api_field: 'nodePools.config.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
645651
- field: 'node_pool.node_config.kubelet_config.max_parallel_image_pulls'
646652
api_field: 'nodePools.config.kubeletConfig.maxParallelImagePulls'
647653
- field: 'node_pool.node_config.kubelet_config.memory_manager.policy'

‎mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2576,6 +2576,73 @@ func TestAccContainerCluster_withKubeletConfig(t *testing.T) {
25762576
})
25772577
}
25782578

2579+
func TestAccContainerCluster_withKubeletConfigShutdownGracePeriod(t *testing.T) {
2580+
t.Parallel()
2581+
2582+
clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
2583+
networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
2584+
subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
2585+
2586+
acctest.VcrTest(t, resource.TestCase{
2587+
PreCheck: func() { acctest.AccTestPreCheck(t) },
2588+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
2589+
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
2590+
Steps: []resource.TestStep{
2591+
{
2592+
Config: testAccContainerCluster_withKubeletConfigShutdownGracePeriod(clusterName, networkName, subnetworkName, 120, 30),
2593+
Check: resource.ComposeTestCheckFunc(
2594+
resource.TestCheckResourceAttr(
2595+
"google_container_cluster.with_kubelet_config_shutdown",
2596+
"node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"),
2597+
resource.TestCheckResourceAttr(
2598+
"google_container_cluster.with_kubelet_config_shutdown",
2599+
"node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"),
2600+
),
2601+
},
2602+
{
2603+
ResourceName: "google_container_cluster.with_kubelet_config_shutdown",
2604+
ImportState: true,
2605+
ImportStateVerify: true,
2606+
ImportStateVerifyIgnore: []string{"deletion_protection"},
2607+
},
2608+
},
2609+
})
2610+
}
2611+
2612+
func TestAccContainerCluster_withInlineNodePoolShutdownGracePeriod(t *testing.T) {
2613+
t.Parallel()
2614+
2615+
clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
2616+
networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
2617+
subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
2618+
2619+
acctest.VcrTest(t, resource.TestCase{
2620+
PreCheck: func() { acctest.AccTestPreCheck(t) },
2621+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
2622+
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
2623+
Steps: []resource.TestStep{
2624+
{
2625+
Config: testAccContainerCluster_withInlineNodePoolShutdownGracePeriod(clusterName, networkName, subnetworkName),
2626+
Check: resource.ComposeTestCheckFunc(
2627+
resource.TestCheckResourceAttr("google_container_cluster.primary", "name", clusterName),
2628+
resource.TestCheckResourceAttr(
2629+
"google_container_cluster.primary",
2630+
"node_pool.0.node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"),
2631+
resource.TestCheckResourceAttr(
2632+
"google_container_cluster.primary",
2633+
"node_pool.0.node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"),
2634+
),
2635+
},
2636+
{
2637+
ResourceName: "google_container_cluster.primary",
2638+
ImportState: true,
2639+
ImportStateVerify: true,
2640+
ImportStateVerifyIgnore: []string{"deletion_protection"},
2641+
},
2642+
},
2643+
})
2644+
}
2645+
25792646
func TestAccContainerCluster_withNodeConfigFastSocket(t *testing.T) {
25802647
t.Parallel()
25812648

@@ -17412,6 +17479,52 @@ resource "google_container_cluster" "with_kubelet_config" {
1741217479
`, clusterName, networkName, subnetworkName, cpuManagerPolicy, memoryManagerPolicy, topologyManagerPolicy, topologyManagerScope)
1741317480
}
1741417481

17482+
func testAccContainerCluster_withKubeletConfigShutdownGracePeriod(clusterName, networkName, subnetworkName string, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds int) string {
17483+
return fmt.Sprintf(`
17484+
resource "google_container_cluster" "with_kubelet_config_shutdown" {
17485+
name = %q
17486+
location = "us-central1-a"
17487+
initial_node_count = 1
17488+
network = %q
17489+
subnetwork = %q
17490+
deletion_protection = false
17491+
17492+
node_config {
17493+
machine_type = "c4-standard-2"
17494+
spot = true
17495+
kubelet_config {
17496+
shutdown_grace_period_seconds = %d
17497+
shutdown_grace_period_critical_pods_seconds = %d
17498+
}
17499+
}
17500+
}
17501+
`, clusterName, networkName, subnetworkName, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds)
17502+
}
17503+
17504+
func testAccContainerCluster_withInlineNodePoolShutdownGracePeriod(clusterName, networkName, subnetworkName string) string {
17505+
return fmt.Sprintf(`
17506+
resource "google_container_cluster" "primary" {
17507+
name = "%s"
17508+
location = "us-central1-a"
17509+
deletion_protection = false
17510+
network = "%s"
17511+
subnetwork = "%s"
17512+
17513+
node_pool {
17514+
name = "primary-pool"
17515+
initial_node_count = 1
17516+
node_config {
17517+
spot = true
17518+
kubelet_config {
17519+
shutdown_grace_period_critical_pods_seconds = 30
17520+
shutdown_grace_period_seconds = 120
17521+
}
17522+
}
17523+
}
17524+
}
17525+
`, clusterName, networkName, subnetworkName)
17526+
}
17527+
1741517528
func testAccContainerCluster_withCpuCfsQuotaPool(clusterName, npName, networkName, subnetworkName string) string {
1741617529
return fmt.Sprintf(`
1741717530
resource "google_container_cluster" "with_kubelet_config" {

‎mmv1/third_party/terraform/services/container/resource_container_node_pool_meta.yaml.tmpl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,10 @@ fields:
198198
api_field: 'config.kubeletConfig.imageMinimumGcAge'
199199
- field: 'node_config.kubelet_config.insecure_kubelet_readonly_port_enabled'
200200
api_field: 'config.kubeletConfig.insecureKubeletReadonlyPortEnabled'
201+
- field: 'node_config.kubelet_config.shutdown_grace_period_seconds'
202+
api_field: 'config.kubeletConfig.shutdownGracePeriodSeconds'
203+
- field: 'node_config.kubelet_config.shutdown_grace_period_critical_pods_seconds'
204+
api_field: 'config.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
201205
- field: 'node_config.kubelet_config.max_parallel_image_pulls'
202206
api_field: 'config.kubeletConfig.maxParallelImagePulls'
203207
- field: 'node_config.kubelet_config.memory_manager.policy'

‎mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -968,6 +968,37 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) {
968968
})
969969
}
970970

971+
func TestAccContainerNodePool_withKubeletConfigShutdownGracePeriod(t *testing.T) {
972+
t.Parallel()
973+
974+
cluster := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
975+
np := fmt.Sprintf("tf-test-np-%s", acctest.RandString(t, 10))
976+
networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
977+
subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
978+
979+
acctest.VcrTest(t, resource.TestCase{
980+
PreCheck: func() { acctest.AccTestPreCheck(t) },
981+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
982+
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
983+
Steps: []resource.TestStep{
984+
{
985+
Config: testAccContainerNodePool_withKubeletConfigShutdownGracePeriod(cluster, np, networkName, subnetworkName, 120, 30),
986+
Check: resource.ComposeTestCheckFunc(
987+
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config_shutdown",
988+
"node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"),
989+
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config_shutdown",
990+
"node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"),
991+
),
992+
},
993+
{
994+
ResourceName: "google_container_node_pool.with_kubelet_config_shutdown",
995+
ImportState: true,
996+
ImportStateVerify: true,
997+
},
998+
},
999+
})
1000+
}
1001+
9711002
func TestAccContainerNodePool_withInvalidKubeletCpuManagerPolicy(t *testing.T) {
9721003
t.Parallel()
9731004
// Unit test, no interactions
@@ -4194,6 +4225,40 @@ resource "google_container_node_pool" "with_kubelet_config" {
41944225
`, cluster, networkName, subnetworkName, np, policy, memoryManagerPolicy, topologyManagerPolicy, topologyManagerScope, quota, period, insecureKubeletReadonlyPortEnabled, podPidsLimit, containerLogMaxSize, containerLogMaxFiles, imageGcLowThresholdPercent, imageGcHighThresholdPercent, imageMinimumGcAge, imageMaximumGcAge, singleProcessOomKill, maxContainerRestart)
41954226
}
41964227

4228+
func testAccContainerNodePool_withKubeletConfigShutdownGracePeriod(cluster, np, networkName, subnetworkName string, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds int) string {
4229+
return fmt.Sprintf(`
4230+
data "google_container_engine_versions" "central1a" {
4231+
location = "us-central1-a"
4232+
}
4233+
4234+
resource "google_container_cluster" "cluster" {
4235+
name = "%s"
4236+
location = "us-central1-a"
4237+
initial_node_count = 1
4238+
min_master_version = data.google_container_engine_versions.central1a.latest_master_version
4239+
deletion_protection = false
4240+
network = "%s"
4241+
subnetwork = "%s"
4242+
}
4243+
4244+
resource "google_container_node_pool" "with_kubelet_config_shutdown" {
4245+
name = "%s"
4246+
location = "us-central1-a"
4247+
cluster = google_container_cluster.cluster.name
4248+
initial_node_count = 1
4249+
node_config {
4250+
machine_type = "c4-standard-2"
4251+
image_type = "COS_CONTAINERD"
4252+
spot = true
4253+
kubelet_config {
4254+
shutdown_grace_period_seconds = %d
4255+
shutdown_grace_period_critical_pods_seconds = %d
4256+
}
4257+
}
4258+
}
4259+
`, cluster, networkName, subnetworkName, np, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds)
4260+
}
4261+
41974262
func testAccContainerNodePool_withLinuxNodeConfig(cluster, np, tcpMem, networkName, subnetworkName string) string {
41984263
linuxNodeConfig := `
41994264
linux_node_config {

‎mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1315,6 +1315,8 @@ workload_identity_config {
13151315

13161316
<a name="nested_node_pool_node_config"></a>The `node_config` block supports:
13171317

1318+
* `kubelet_config` - (Optional) Node kubelet configs. Structure is [documented below](#nested_kubelet_config).
1319+
13181320
* `taint_config` - (Optional) Taint configuration for the node pool. Structure is [documented below](#nested_node_pool_node_config_taint_config).
13191321

13201322
<a name="nested_node_pool_node_config_taint_config"></a>The `taint_config` block supports:
@@ -1324,7 +1326,7 @@ workload_identity_config {
13241326
<a name="nested_node_pool_auto_config"></a>The `node_pool_auto_config` block supports:
13251327

13261328
* `node_kubelet_config` - (Optional) Kubelet configuration for Autopilot clusters. Currently, only `insecure_kubelet_readonly_port_enabled` is supported here.
1327-
Structure is [documented below](#nested_node_kubelet_config).
1329+
Structure is [documented below](#nested_node_kubelet_config).
13281330

13291331
* `resource_manager_tags` - (Optional) A map of resource manager tag keys and values to be attached to the nodes for managing Compute Engine firewalls using Network Firewall Policies. Tags must be according to specifications found [here](https://cloud.google.com/vpc/docs/tags-firewalls-overview#specifications). A maximum of 5 tag key-value pairs can be specified. Existing tags will be replaced with new values. Tags must be in one of the following formats ([KEY]=[VALUE]) 1. `tagKeys/{tag_key_id}=tagValues/{tag_value_id}` 2. `{org_id}/{tag_key_name}={tag_value_name}` 3. `{project_id}/{tag_key_name}={tag_value_name}`.
13301332

@@ -1653,6 +1655,10 @@ those in the Guaranteed QoS class, by influencing NUMA affinity. Structure is [d
16531655

16541656
* `crash_loop_back_off` - (Optional) Contains configuration options to modify node-level parameters for container restart behavior. Structure is [documented below](#nested_crash_loop_back_off).
16551657

1658+
* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
1659+
1660+
* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
1661+
16561662
<a name="nested_eviction_soft"></a>The `eviction_soft` block supports:
16571663

16581664
* `memory_available` - (Optional) Defines quantity of soft eviction threshold for memory.available. The value must be a quantity, such as `"100Mi"`. The value must be greater than or equal to the GKE default hard eviction threshold of `"100Mi"` and less than 50% of machine memory.

‎mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,8 @@ cluster.
329329

330330
<a name="nested_node_config"></a>The `node_config` block supports:
331331

332+
* `kubelet_config` - (Optional) Node kubelet configs. Structure is [documented below](#nested_kubelet_config).
333+
332334
* `taint_config` - (Optional) Taint configuration for the node pool. Structure is [documented below](#nested_node_config_taint_config).
333335

334336
<a name="nested_node_config_taint_config"></a>The `taint_config` block supports:
@@ -363,6 +365,12 @@ In addition to the arguments listed above, the following computed attributes are
363365
- `update` - (Default `60 minutes`) Used for updates to node pools
364366
- `delete` - (Default `60 minutes`) Used for removing node pools.
365367

368+
<a name="nested_kubelet_config"></a>The `kubelet_config` block supports:
369+
370+
* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
371+
372+
* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
373+
366374
## Import
367375

368376
Node pools can be imported using the `project`, `location`, `cluster` and `name`. If

0 commit comments

Comments
 (0)