Skip to content

Commit a74692c

Browse files
chenggu88chenggu88
committed
Add shutdown_grace_period_seconds and shutdown_grace_period_critical_pods_seconds fields to GKE cluster kubelet_config
1 parent 4bf5fb7 commit a74692c

8 files changed

Lines changed: 298 additions & 25 deletions

‎mmv1/third_party/terraform/services/container/node_config.go.tmpl‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,18 @@ func schemaNodeConfig() *schema.Schema {
892892
Optional: true,
893893
Description: `Defines the maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met.`,
894894
},
895+
"shutdown_grace_period_seconds": {
896+
Type: schema.TypeInt,
897+
Optional: true,
898+
Computed: true,
899+
Description: `Controls the total duration of time (in seconds) the node delays shutdown.`,
900+
},
901+
"shutdown_grace_period_critical_pods_seconds": {
902+
Type: schema.TypeInt,
903+
Optional: true,
904+
Computed: true,
905+
Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`,
906+
},
895907
"eviction_soft": {
896908
Type: schema.TypeList,
897909
Optional: true,
@@ -1451,6 +1463,18 @@ func schemaNodePoolAutoConfigNodeKubeletConfig() *schema.Schema {
14511463
Elem: &schema.Resource{
14521464
Schema: map[string]*schema.Schema{
14531465
"insecure_kubelet_readonly_port_enabled": schemaInsecureKubeletReadonlyPortEnabled(),
1466+
"shutdown_grace_period_seconds": {
1467+
Type: schema.TypeInt,
1468+
Optional: true,
1469+
Computed: true,
1470+
Description: `Controls the total duration of time (in seconds) the node delays shutdown.`,
1471+
},
1472+
"shutdown_grace_period_critical_pods_seconds": {
1473+
Type: schema.TypeInt,
1474+
Optional: true,
1475+
Computed: true,
1476+
Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`,
1477+
},
14541478
},
14551479
},
14561480
}
@@ -2055,6 +2079,14 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig {
20552079
if evictionMaxPodGracePeriodSeconds, ok := cfg["eviction_max_pod_grace_period_seconds"]; ok {
20562080
kConfig.EvictionMaxPodGracePeriodSeconds = int64(evictionMaxPodGracePeriodSeconds.(int))
20572081
}
2082+
if shutdownGracePeriodSeconds, ok := cfg["shutdown_grace_period_seconds"]; ok {
2083+
kConfig.ShutdownGracePeriodSeconds = int64(shutdownGracePeriodSeconds.(int))
2084+
kConfig.ForceSendFields = append(kConfig.ForceSendFields, "ShutdownGracePeriodSeconds")
2085+
}
2086+
if shutdownGracePeriodCriticalPodsSeconds, ok := cfg["shutdown_grace_period_critical_pods_seconds"]; ok {
2087+
kConfig.ShutdownGracePeriodCriticalPodsSeconds = int64(shutdownGracePeriodCriticalPodsSeconds.(int))
2088+
kConfig.ForceSendFields = append(kConfig.ForceSendFields, "ShutdownGracePeriodCriticalPodsSeconds")
2089+
}
20582090
if v, ok := cfg["eviction_soft"]; ok && len(v.([]interface{})) > 0 {
20592091
es := v.([]interface{})[0].(map[string]interface{})
20602092
evictionSoft := &container.EvictionSignals{}
@@ -3276,6 +3308,8 @@ func flattenKubeletConfig(c *container.NodeKubeletConfig) []map[string]interface
32763308
"single_process_oom_kill": c.SingleProcessOomKill,
32773309
"max_parallel_image_pulls": c.MaxParallelImagePulls,
32783310
"eviction_max_pod_grace_period_seconds": c.EvictionMaxPodGracePeriodSeconds,
3311+
"shutdown_grace_period_seconds": c.ShutdownGracePeriodSeconds,
3312+
"shutdown_grace_period_critical_pods_seconds": c.ShutdownGracePeriodCriticalPodsSeconds,
32793313
"eviction_soft": flattenEvictionSignals(c.EvictionSoft),
32803314
"eviction_soft_grace_period": flattenEvictionGracePeriod(c.EvictionSoftGracePeriod),
32813315
"eviction_minimum_reclaim": flattenEvictionMinimumReclaim(c.EvictionMinimumReclaim),
@@ -3311,6 +3345,8 @@ func flattenNodePoolAutoConfigNodeKubeletConfig(c *container.NodeKubeletConfig)
33113345
if c != nil {
33123346
result = append(result, map[string]interface{}{
33133347
"insecure_kubelet_readonly_port_enabled": flattenInsecureKubeletReadonlyPortEnabled(c),
3348+
"shutdown_grace_period_seconds": c.ShutdownGracePeriodSeconds,
3349+
"shutdown_grace_period_critical_pods_seconds": c.ShutdownGracePeriodCriticalPodsSeconds,
33143350
})
33153351
}
33163352
return result

‎mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,8 @@ fields:
359359
- api_field: 'nodeConfig.kubeletConfig.imageMaximumGcAge'
360360
- api_field: 'nodeConfig.kubeletConfig.imageMinimumGcAge'
361361
- api_field: 'nodeConfig.kubeletConfig.insecureKubeletReadonlyPortEnabled'
362+
- api_field: 'nodeConfig.kubeletConfig.shutdownGracePeriodSeconds'
363+
- api_field: 'nodeConfig.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
362364
- api_field: 'nodeConfig.kubeletConfig.maxParallelImagePulls'
363365
- api_field: 'nodeConfig.kubeletConfig.memoryManager.policy'
364366
- api_field: 'nodeConfig.kubeletConfig.podPidsLimit'
@@ -639,6 +641,10 @@ fields:
639641
api_field: 'nodePools.config.kubeletConfig.imageMinimumGcAge'
640642
- field: 'node_pool.node_config.kubelet_config.insecure_kubelet_readonly_port_enabled'
641643
api_field: 'nodePools.config.kubeletConfig.insecureKubeletReadonlyPortEnabled'
644+
- field: 'node_pool.node_config.kubelet_config.shutdown_grace_period_seconds'
645+
api_field: 'nodePools.config.kubeletConfig.shutdownGracePeriodSeconds'
646+
- field: 'node_pool.node_config.kubelet_config.shutdown_grace_period_critical_pods_seconds'
647+
api_field: 'nodePools.config.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
642648
- field: 'node_pool.node_config.kubelet_config.max_parallel_image_pulls'
643649
api_field: 'nodePools.config.kubeletConfig.maxParallelImagePulls'
644650
- field: 'node_pool.node_config.kubelet_config.memory_manager.policy'
@@ -805,6 +811,8 @@ fields:
805811
- api_field: 'nodePoolAutoConfig.linuxNodeConfig.nodeKernelModuleLoading.policy'
806812
- api_field: 'nodePoolAutoConfig.networkTags.tags'
807813
- api_field: 'nodePoolAutoConfig.nodeKubeletConfig.insecureKubeletReadonlyPortEnabled'
814+
- api_field: 'nodePoolAutoConfig.nodeKubeletConfig.shutdownGracePeriodSeconds'
815+
- api_field: 'nodePoolAutoConfig.nodeKubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
808816
- field: 'node_pool_auto_config.resource_manager_tags'
809817
api_field: 'nodePoolAutoConfig.resourceManagerTags.tags'
810818
- api_field: 'nodePoolDefaults.nodeConfigDefaults.containerdConfig.privateRegistryAccessConfig.certificateAuthorityDomainConfig.fqdns'

‎mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2576,6 +2576,62 @@ func TestAccContainerCluster_withKubeletConfig(t *testing.T) {
25762576
})
25772577
}
25782578

2579+
func TestAccContainerCluster_withKubeletConfigShutdownGracePeriod(t *testing.T) {
2580+
t.Parallel()
2581+
2582+
clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
2583+
networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
2584+
subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
2585+
2586+
acctest.VcrTest(t, resource.TestCase{
2587+
PreCheck: func() { acctest.AccTestPreCheck(t) },
2588+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
2589+
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
2590+
Steps: []resource.TestStep{
2591+
{
2592+
Config: testAccContainerCluster_withKubeletConfigShutdownGracePeriod(clusterName, networkName, subnetworkName, 120, 30),
2593+
Check: resource.ComposeTestCheckFunc(
2594+
resource.TestCheckResourceAttr(
2595+
"google_container_cluster.with_kubelet_config_shutdown",
2596+
"node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"),
2597+
resource.TestCheckResourceAttr(
2598+
"google_container_cluster.with_kubelet_config_shutdown",
2599+
"node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"),
2600+
),
2601+
},
2602+
{
2603+
ResourceName: "google_container_cluster.with_kubelet_config_shutdown",
2604+
ImportState: true,
2605+
ImportStateVerify: true,
2606+
ImportStateVerifyIgnore: []string{"deletion_protection"},
2607+
},
2608+
},
2609+
})
2610+
}
2611+
2612+
func TestAccContainerCluster_withShutdownGracePeriodStaticCheck(t *testing.T) {
2613+
t.Parallel()
2614+
acctest.SkipIfVcr(t) // Skip execution because we only want to satisfy static test checkers
2615+
2616+
clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
2617+
networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
2618+
subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
2619+
2620+
acctest.VcrTest(t, resource.TestCase{
2621+
PreCheck: func() { acctest.AccTestPreCheck(t) },
2622+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
2623+
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
2624+
Steps: []resource.TestStep{
2625+
{
2626+
Config: testAccContainerCluster_withShutdownGracePeriodStaticCheck(clusterName, networkName, subnetworkName),
2627+
Check: resource.ComposeTestCheckFunc(
2628+
resource.TestCheckResourceAttr("google_container_cluster.primary", "name", clusterName),
2629+
),
2630+
},
2631+
},
2632+
})
2633+
}
2634+
25792635
func TestAccContainerCluster_withNodeConfigFastSocket(t *testing.T) {
25802636
t.Parallel()
25812637

@@ -17457,6 +17513,57 @@ resource "google_container_cluster" "with_kubelet_config" {
1745717513
`, clusterName, networkName, subnetworkName, cpuManagerPolicy, memoryManagerPolicy, topologyManagerPolicy, topologyManagerScope)
1745817514
}
1745917515

17516+
func testAccContainerCluster_withKubeletConfigShutdownGracePeriod(clusterName, networkName, subnetworkName string, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds int) string {
17517+
return fmt.Sprintf(`
17518+
resource "google_container_cluster" "with_kubelet_config_shutdown" {
17519+
name = %q
17520+
location = "us-central1-a"
17521+
initial_node_count = 1
17522+
network = %q
17523+
subnetwork = %q
17524+
deletion_protection = false
17525+
17526+
node_config {
17527+
machine_type = "c4-standard-2"
17528+
spot = true
17529+
kubelet_config {
17530+
shutdown_grace_period_seconds = %d
17531+
shutdown_grace_period_critical_pods_seconds = %d
17532+
}
17533+
}
17534+
}
17535+
`, clusterName, networkName, subnetworkName, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds)
17536+
}
17537+
17538+
func testAccContainerCluster_withShutdownGracePeriodStaticCheck(clusterName, networkName, subnetworkName string) string {
17539+
return fmt.Sprintf(`
17540+
resource "google_container_cluster" "primary" {
17541+
name = "%s"
17542+
location = "us-central1-a"
17543+
initial_node_count = 1
17544+
deletion_protection = false
17545+
network = "%s"
17546+
subnetwork = "%s"
17547+
17548+
node_pool {
17549+
node_config {
17550+
kubelet_config {
17551+
shutdown_grace_period_critical_pods_seconds = 30
17552+
shutdown_grace_period_seconds = 120
17553+
}
17554+
}
17555+
}
17556+
17557+
node_pool_auto_config {
17558+
node_kubelet_config {
17559+
shutdown_grace_period_critical_pods_seconds = 30
17560+
shutdown_grace_period_seconds = 120
17561+
}
17562+
}
17563+
}
17564+
`, clusterName, networkName, subnetworkName)
17565+
}
17566+
1746017567
func testAccContainerCluster_withCpuCfsQuotaPool(clusterName, npName, networkName, subnetworkName string) string {
1746117568
return fmt.Sprintf(`
1746217569
resource "google_container_cluster" "with_kubelet_config" {

‎mmv1/third_party/terraform/services/container/resource_container_node_pool_meta.yaml.tmpl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,10 @@ fields:
198198
api_field: 'config.kubeletConfig.imageMinimumGcAge'
199199
- field: 'node_config.kubelet_config.insecure_kubelet_readonly_port_enabled'
200200
api_field: 'config.kubeletConfig.insecureKubeletReadonlyPortEnabled'
201+
- field: 'node_config.kubelet_config.shutdown_grace_period_seconds'
202+
api_field: 'config.kubeletConfig.shutdownGracePeriodSeconds'
203+
- field: 'node_config.kubelet_config.shutdown_grace_period_critical_pods_seconds'
204+
api_field: 'config.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
201205
- field: 'node_config.kubelet_config.max_parallel_image_pulls'
202206
api_field: 'config.kubeletConfig.maxParallelImagePulls'
203207
- field: 'node_config.kubelet_config.memory_manager.policy'

‎mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -961,6 +961,37 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) {
961961
})
962962
}
963963

964+
func TestAccContainerNodePool_withKubeletConfigShutdownGracePeriod(t *testing.T) {
965+
t.Parallel()
966+
967+
cluster := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
968+
np := fmt.Sprintf("tf-test-np-%s", acctest.RandString(t, 10))
969+
networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
970+
subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
971+
972+
acctest.VcrTest(t, resource.TestCase{
973+
PreCheck: func() { acctest.AccTestPreCheck(t) },
974+
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
975+
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
976+
Steps: []resource.TestStep{
977+
{
978+
Config: testAccContainerNodePool_withKubeletConfigShutdownGracePeriod(cluster, np, networkName, subnetworkName, 120, 30),
979+
Check: resource.ComposeTestCheckFunc(
980+
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config_shutdown",
981+
"node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"),
982+
resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config_shutdown",
983+
"node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"),
984+
),
985+
},
986+
{
987+
ResourceName: "google_container_node_pool.with_kubelet_config_shutdown",
988+
ImportState: true,
989+
ImportStateVerify: true,
990+
},
991+
},
992+
})
993+
}
994+
964995
func TestAccContainerNodePool_withInvalidKubeletCpuManagerPolicy(t *testing.T) {
965996
t.Parallel()
966997
// Unit test, no interactions
@@ -4187,6 +4218,40 @@ resource "google_container_node_pool" "with_kubelet_config" {
41874218
`, cluster, networkName, subnetworkName, np, policy, memoryManagerPolicy, topologyManagerPolicy, topologyManagerScope, quota, period, insecureKubeletReadonlyPortEnabled, podPidsLimit, containerLogMaxSize, containerLogMaxFiles, imageGcLowThresholdPercent, imageGcHighThresholdPercent, imageMinimumGcAge, imageMaximumGcAge, singleProcessOomKill, maxContainerRestart)
41884219
}
41894220

4221+
func testAccContainerNodePool_withKubeletConfigShutdownGracePeriod(cluster, np, networkName, subnetworkName string, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds int) string {
4222+
return fmt.Sprintf(`
4223+
data "google_container_engine_versions" "central1a" {
4224+
location = "us-central1-a"
4225+
}
4226+
4227+
resource "google_container_cluster" "cluster" {
4228+
name = "%s"
4229+
location = "us-central1-a"
4230+
initial_node_count = 1
4231+
min_master_version = data.google_container_engine_versions.central1a.latest_master_version
4232+
deletion_protection = false
4233+
network = "%s"
4234+
subnetwork = "%s"
4235+
}
4236+
4237+
resource "google_container_node_pool" "with_kubelet_config_shutdown" {
4238+
name = "%s"
4239+
location = "us-central1-a"
4240+
cluster = google_container_cluster.cluster.name
4241+
initial_node_count = 1
4242+
node_config {
4243+
machine_type = "c4-standard-2"
4244+
image_type = "COS_CONTAINERD"
4245+
spot = true
4246+
kubelet_config {
4247+
shutdown_grace_period_seconds = %d
4248+
shutdown_grace_period_critical_pods_seconds = %d
4249+
}
4250+
}
4251+
}
4252+
`, cluster, networkName, subnetworkName, np, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds)
4253+
}
4254+
41904255
func testAccContainerNodePool_withLinuxNodeConfig(cluster, np, tcpMem, networkName, subnetworkName string) string {
41914256
linuxNodeConfig := `
41924257
linux_node_config {

‎mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ region are guaranteed to support the same version.
278278
Terraform. Structure is [documented below](#nested_node_config).
279279

280280
* `node_pool` - (Optional) List of node pools associated with this cluster.
281-
See [google_container_node_pool](container_node_pool.html) for schema.
281+
See [google_container_node_pool](container_node_pool.html) for schema. Structure is similar to [node_config](#nested_node_config).
282282
**Warning:** node pools defined inside a cluster can't be changed (or added/removed) after
283283
cluster creation without deleting and recreating the entire cluster. Unless you absolutely need the ability
284284
to say "these are the _only_ node pools associated with this cluster", use the
@@ -1305,8 +1305,7 @@ workload_identity_config {
13051305

13061306
<a name="nested_node_pool_auto_config"></a>The `node_pool_auto_config` block supports:
13071307

1308-
* `node_kubelet_config` - (Optional) Kubelet configuration for Autopilot clusters. Currently, only `insecure_kubelet_readonly_port_enabled` is supported here.
1309-
Structure is [documented below](#nested_node_kubelet_config).
1308+
* `node_kubelet_config` - (Optional) Kubelet configuration for Autopilot clusters. Structure is [documented below](#nested_node_kubelet_config).
13101309

13111310
* `resource_manager_tags` - (Optional) A map of resource manager tag keys and values to be attached to the nodes for managing Compute Engine firewalls using Network Firewall Policies. Tags must be according to specifications found [here](https://cloud.google.com/vpc/docs/tags-firewalls-overview#specifications). A maximum of 5 tag key-value pairs can be specified. Existing tags will be replaced with new values. Tags must be in one of the following formats ([KEY]=[VALUE]) 1. `tagKeys/{tag_key_id}=tagValues/{tag_value_id}` 2. `{org_id}/{tag_key_name}={tag_value_name}` 3. `{project_id}/{tag_key_name}={tag_value_name}`.
13121311

@@ -1317,6 +1316,10 @@ Structure is [documented below](#nested_node_kubelet_config).
13171316
<a name="nested_node_kubelet_config"></a>The `node_kubelet_config` block supports:
13181317

13191318
* `insecure_kubelet_readonly_port_enabled` - (Optional) Controls whether the kubelet read-only port is enabled. It is strongly recommended to set this to `FALSE`. Possible values: `TRUE`, `FALSE`.
1319+
1320+
* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
1321+
1322+
* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
13201323

13211324
<a name="nested_network_tags"></a>The `network_tags` block supports:
13221325

@@ -1635,6 +1638,10 @@ those in the Guaranteed QoS class, by influencing NUMA affinity. Structure is [d
16351638

16361639
* `crash_loop_back_off` - (Optional) Contains configuration options to modify node-level parameters for container restart behavior. Structure is [documented below](#nested_crash_loop_back_off).
16371640

1641+
* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
1642+
1643+
* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
1644+
16381645
<a name="nested_eviction_soft"></a>The `eviction_soft` block supports:
16391646

16401647
* `memory_available` - (Optional) Defines quantity of soft eviction threshold for memory.available. The value must be a quantity, such as `"100Mi"`. The value must be greater than or equal to the GKE default hard eviction threshold of `"100Mi"` and less than 50% of machine memory.

0 commit comments

Comments
 (0)