Add shutdown_grace_period_seconds and shutdown_grace_period_critical_pods_seconds fields to GKE cluster kubelet_config

mmv1/third_party/terraform/services/container/node_config.go.tmpl

@@ -912,6 +912,18 @@ func schemaNodeConfig() *schema.Schema {
                                                                 Optional:    true,
                                                                 Description: `Defines the maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met.`,
                                                         },
+							"shutdown_grace_period_seconds": {
+								Type:        schema.TypeInt,
+								Optional:    true,
+								Computed:    true,
+								Description: `Controls the total duration of time (in seconds) the node delays shutdown.`,
+							},
+							"shutdown_grace_period_critical_pods_seconds": {
+								Type:        schema.TypeInt,
+								Optional:    true,
+								Computed:    true,
+								Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`,
+							},
 							"eviction_soft": {
 								Type:        schema.TypeList,
 								Optional:    true,
@@ -1875,6 +1887,21 @@ func expandNodeConfig(d *schema.ResourceData, prefix string, v interface{}) *con
 			}
 		}
 		// end cpu_cfs_quota fix
+
+		// start shutdown_grace_period ForceSendFields fix
+		if vNC := rawConfigNPRoot.GetAttr("node_config"); vNC.LengthInt() > 0 {
+			if vKC := vNC.Index(cty.NumberIntVal(0)).GetAttr("kubelet_config"); vKC.LengthInt() > 0 {
+				vSGP := vKC.Index(cty.NumberIntVal(0)).GetAttr("shutdown_grace_period_seconds")
+				if vSGP != cty.NullVal(cty.Number) && !vSGP.IsNull() {
+					nc.KubeletConfig.ForceSendFields = append(nc.KubeletConfig.ForceSendFields, "ShutdownGracePeriodSeconds")
+				}
+				vSGPC := vKC.Index(cty.NumberIntVal(0)).GetAttr("shutdown_grace_period_critical_pods_seconds")
+				if vSGPC != cty.NullVal(cty.Number) && !vSGPC.IsNull() {
+					nc.KubeletConfig.ForceSendFields = append(nc.KubeletConfig.ForceSendFields, "ShutdownGracePeriodCriticalPodsSeconds")
+				}
+			}
+		}
+		// end shutdown_grace_period ForceSendFields fix
 	}
 
 	if v, ok := nodeConfig["linux_node_config"]; ok {
@@ -2079,6 +2106,12 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig {
 	if evictionMaxPodGracePeriodSeconds, ok := cfg["eviction_max_pod_grace_period_seconds"]; ok {
                 kConfig.EvictionMaxPodGracePeriodSeconds = int64(evictionMaxPodGracePeriodSeconds.(int))
         }
+	if shutdownGracePeriodSeconds, ok := cfg["shutdown_grace_period_seconds"]; ok {
+		kConfig.ShutdownGracePeriodSeconds = int64(shutdownGracePeriodSeconds.(int))
+	}
+	if shutdownGracePeriodCriticalPodsSeconds, ok := cfg["shutdown_grace_period_critical_pods_seconds"]; ok {
+		kConfig.ShutdownGracePeriodCriticalPodsSeconds = int64(shutdownGracePeriodCriticalPodsSeconds.(int))
+	}
 	if v, ok := cfg["eviction_soft"]; ok && len(v.([]interface{})) > 0 {
 		es := v.([]interface{})[0].(map[string]interface{})
 		evictionSoft := &container.EvictionSignals{}
@@ -3323,6 +3356,8 @@ func flattenKubeletConfig(c *container.NodeKubeletConfig) []map[string]interface
 			"single_process_oom_kill":                c.SingleProcessOomKill,
 			"max_parallel_image_pulls":               c.MaxParallelImagePulls,
 			"eviction_max_pod_grace_period_seconds":  c.EvictionMaxPodGracePeriodSeconds,
+			"shutdown_grace_period_seconds":          c.ShutdownGracePeriodSeconds,
+			"shutdown_grace_period_critical_pods_seconds": c.ShutdownGracePeriodCriticalPodsSeconds,
 			"eviction_soft":                          flattenEvictionSignals(c.EvictionSoft),
 			"eviction_soft_grace_period":             flattenEvictionGracePeriod(c.EvictionSoftGracePeriod),
 			"eviction_minimum_reclaim":               flattenEvictionMinimumReclaim(c.EvictionMinimumReclaim),

mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl

@@ -359,6 +359,8 @@ fields:
   - api_field: 'nodeConfig.kubeletConfig.imageMaximumGcAge'
   - api_field: 'nodeConfig.kubeletConfig.imageMinimumGcAge'
   - api_field: 'nodeConfig.kubeletConfig.insecureKubeletReadonlyPortEnabled'
+  - api_field: 'nodeConfig.kubeletConfig.shutdownGracePeriodSeconds'
+  - api_field: 'nodeConfig.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
   - api_field: 'nodeConfig.kubeletConfig.maxParallelImagePulls'
   - api_field: 'nodeConfig.kubeletConfig.memoryManager.policy'
   - api_field: 'nodeConfig.kubeletConfig.podPidsLimit'
@@ -642,6 +644,10 @@ fields:
     api_field: 'nodePools.config.kubeletConfig.imageMinimumGcAge'
   - field: 'node_pool.node_config.kubelet_config.insecure_kubelet_readonly_port_enabled'
     api_field: 'nodePools.config.kubeletConfig.insecureKubeletReadonlyPortEnabled'
+  - field: 'node_pool.node_config.kubelet_config.shutdown_grace_period_seconds'
+    api_field: 'nodePools.config.kubeletConfig.shutdownGracePeriodSeconds'
+  - field: 'node_pool.node_config.kubelet_config.shutdown_grace_period_critical_pods_seconds'
+    api_field: 'nodePools.config.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
   - field: 'node_pool.node_config.kubelet_config.max_parallel_image_pulls'
     api_field: 'nodePools.config.kubeletConfig.maxParallelImagePulls'
   - field: 'node_pool.node_config.kubelet_config.memory_manager.policy'

mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl

@@ -2576,6 +2576,73 @@ func TestAccContainerCluster_withKubeletConfig(t *testing.T) {
 	})
 }
 
+func TestAccContainerCluster_withKubeletConfigShutdownGracePeriod(t *testing.T) {
+	t.Parallel()
+
+	clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+	networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
+	subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_withKubeletConfigShutdownGracePeriod(clusterName, networkName, subnetworkName, 120, 30),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr(
+						"google_container_cluster.with_kubelet_config_shutdown",
+						"node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"),
+					resource.TestCheckResourceAttr(
+						"google_container_cluster.with_kubelet_config_shutdown",
+						"node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"),
+				),
+			},
+			{
+				ResourceName:            "google_container_cluster.with_kubelet_config_shutdown",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+		},
+	})
+}
+
+func TestAccContainerCluster_withInlineNodePoolShutdownGracePeriod(t *testing.T) {
+	t.Parallel()
+
+	clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+	networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
+	subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_withInlineNodePoolShutdownGracePeriod(clusterName, networkName, subnetworkName),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr("google_container_cluster.primary", "name", clusterName),
+					resource.TestCheckResourceAttr(
+						"google_container_cluster.primary",
+						"node_pool.0.node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"),
+					resource.TestCheckResourceAttr(
+						"google_container_cluster.primary",
+						"node_pool.0.node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"),
+				),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+		},
+	})
+}
+
 func TestAccContainerCluster_withNodeConfigFastSocket(t *testing.T) {
 	t.Parallel()
 
@@ -17412,6 +17479,52 @@ resource "google_container_cluster" "with_kubelet_config" {
 `, clusterName, networkName, subnetworkName, cpuManagerPolicy, memoryManagerPolicy, topologyManagerPolicy, topologyManagerScope)
 }
 
+func testAccContainerCluster_withKubeletConfigShutdownGracePeriod(clusterName, networkName, subnetworkName string, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds int) string {
+	return fmt.Sprintf(`
+resource "google_container_cluster" "with_kubelet_config_shutdown" {
+  name               = %q
+  location           = "us-central1-a"
+  initial_node_count = 1
+  network            = %q
+  subnetwork         = %q
+  deletion_protection = false
+
+  node_config {
+    machine_type = "c4-standard-2"
+    spot         = true
+    kubelet_config {
+      shutdown_grace_period_seconds = %d
+      shutdown_grace_period_critical_pods_seconds = %d
+    }
+  }
+}
+`, clusterName, networkName, subnetworkName, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds)
+}
+
+func testAccContainerCluster_withInlineNodePoolShutdownGracePeriod(clusterName, networkName, subnetworkName string) string {
+	return fmt.Sprintf(`
+resource "google_container_cluster" "primary" {
+  name               = "%s"
+  location           = "us-central1-a"
+  deletion_protection = false
+  network    = "%s"
+  subnetwork    = "%s"
+
+  node_pool {
+    name = "primary-pool"
+    initial_node_count = 1
+    node_config {
+      spot = true
+      kubelet_config {
+        shutdown_grace_period_critical_pods_seconds = 30
+        shutdown_grace_period_seconds               = 120
+      }
+    }
+  }
+}
+`, clusterName, networkName, subnetworkName)
+}
+
 func testAccContainerCluster_withCpuCfsQuotaPool(clusterName, npName, networkName, subnetworkName string) string {
 	return fmt.Sprintf(`
 resource "google_container_cluster" "with_kubelet_config" {

mmv1/third_party/terraform/services/container/resource_container_node_pool_meta.yaml.tmpl

@@ -198,6 +198,10 @@ fields:
     api_field: 'config.kubeletConfig.imageMinimumGcAge'
   - field: 'node_config.kubelet_config.insecure_kubelet_readonly_port_enabled'
     api_field: 'config.kubeletConfig.insecureKubeletReadonlyPortEnabled'
+  - field: 'node_config.kubelet_config.shutdown_grace_period_seconds'
+    api_field: 'config.kubeletConfig.shutdownGracePeriodSeconds'
+  - field: 'node_config.kubelet_config.shutdown_grace_period_critical_pods_seconds'
+    api_field: 'config.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds'
   - field: 'node_config.kubelet_config.max_parallel_image_pulls'
     api_field: 'config.kubeletConfig.maxParallelImagePulls'
   - field: 'node_config.kubelet_config.memory_manager.policy'

mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl

@@ -968,6 +968,37 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) {
 	})
 }
 
+func TestAccContainerNodePool_withKubeletConfigShutdownGracePeriod(t *testing.T) {
+	t.Parallel()
+
+	cluster := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+	np := fmt.Sprintf("tf-test-np-%s", acctest.RandString(t, 10))
+	networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster")
+	subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName)
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerNodePool_withKubeletConfigShutdownGracePeriod(cluster, np, networkName, subnetworkName, 120, 30),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config_shutdown",
+						"node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"),
+					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config_shutdown",
+						"node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"),
+				),
+			},
+			{
+				ResourceName:      "google_container_node_pool.with_kubelet_config_shutdown",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+		},
+	})
+}
+
 func TestAccContainerNodePool_withInvalidKubeletCpuManagerPolicy(t *testing.T) {
 	t.Parallel()
 	// Unit test, no interactions
@@ -4194,6 +4225,40 @@ resource "google_container_node_pool" "with_kubelet_config" {
 `, cluster, networkName, subnetworkName, np, policy, memoryManagerPolicy, topologyManagerPolicy, topologyManagerScope, quota, period, insecureKubeletReadonlyPortEnabled, podPidsLimit, containerLogMaxSize, containerLogMaxFiles, imageGcLowThresholdPercent, imageGcHighThresholdPercent, imageMinimumGcAge, imageMaximumGcAge, singleProcessOomKill, maxContainerRestart)
 }
 
+func testAccContainerNodePool_withKubeletConfigShutdownGracePeriod(cluster, np, networkName, subnetworkName string, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds int) string {
+	return fmt.Sprintf(`
+data "google_container_engine_versions" "central1a" {
+  location = "us-central1-a"
+}
+
+resource "google_container_cluster" "cluster" {
+  name               = "%s"
+  location           = "us-central1-a"
+  initial_node_count = 1
+  min_master_version = data.google_container_engine_versions.central1a.latest_master_version
+  deletion_protection = false
+  network    = "%s"
+  subnetwork    = "%s"
+}
+
+resource "google_container_node_pool" "with_kubelet_config_shutdown" {
+  name               = "%s"
+  location           = "us-central1-a"
+  cluster            = google_container_cluster.cluster.name
+  initial_node_count = 1
+  node_config {
+    machine_type       = "c4-standard-2"
+    image_type = "COS_CONTAINERD"
+    spot = true
+    kubelet_config {
+      shutdown_grace_period_seconds = %d
+      shutdown_grace_period_critical_pods_seconds = %d
+    }
+  }
+}
+`, cluster, networkName, subnetworkName, np, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds)
+}
+
 func testAccContainerNodePool_withLinuxNodeConfig(cluster, np, tcpMem, networkName, subnetworkName string) string {
 	linuxNodeConfig := `
     linux_node_config {

mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown

@@ -1315,6 +1315,8 @@ workload_identity_config {
 
 <a name="nested_node_pool_node_config"></a>The `node_config` block supports:
 
+* `kubelet_config` - (Optional) Node kubelet configs. Structure is [documented below](#nested_kubelet_config).
+
 * `taint_config` - (Optional) Taint configuration for the node pool. Structure is [documented below](#nested_node_pool_node_config_taint_config).
 
 <a name="nested_node_pool_node_config_taint_config"></a>The `taint_config` block supports:
@@ -1324,7 +1326,7 @@ workload_identity_config {
 <a name="nested_node_pool_auto_config"></a>The `node_pool_auto_config` block supports:
 
 * `node_kubelet_config` - (Optional) Kubelet configuration for Autopilot clusters. Currently, only `insecure_kubelet_readonly_port_enabled` is supported here.
-Structure is [documented below](#nested_node_kubelet_config).
+    Structure is [documented below](#nested_node_kubelet_config).
 
 * `resource_manager_tags` - (Optional) A map of resource manager tag keys and values to be attached to the nodes for managing Compute Engine firewalls using Network Firewall Policies. Tags must be according to specifications found [here](https://cloud.google.com/vpc/docs/tags-firewalls-overview#specifications). A maximum of 5 tag key-value pairs can be specified. Existing tags will be replaced with new values. Tags must be in one of the following formats ([KEY]=[VALUE]) 1. `tagKeys/{tag_key_id}=tagValues/{tag_value_id}` 2. `{org_id}/{tag_key_name}={tag_value_name}` 3. `{project_id}/{tag_key_name}={tag_value_name}`.
 
@@ -1653,6 +1655,10 @@ those in the Guaranteed QoS class, by influencing NUMA affinity. Structure is [d
 
 * `crash_loop_back_off` - (Optional) Contains configuration options to modify node-level parameters for container restart behavior. Structure is [documented below](#nested_crash_loop_back_off).
 
+* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
+
+* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
+
 <a name="nested_eviction_soft"></a>The `eviction_soft` block supports:
 
 * `memory_available` - (Optional) Defines quantity of soft eviction threshold for memory.available. The value must be a quantity, such as `"100Mi"`. The value must be greater than or equal to the GKE default hard eviction threshold of `"100Mi"` and less than 50% of machine memory.

mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown

@@ -329,6 +329,8 @@ cluster.
 
 <a name="nested_node_config"></a>The `node_config` block supports:
 
+* `kubelet_config` - (Optional) Node kubelet configs. Structure is [documented below](#nested_kubelet_config).
+
 * `taint_config` - (Optional) Taint configuration for the node pool. Structure is [documented below](#nested_node_config_taint_config).
 
 <a name="nested_node_config_taint_config"></a>The `taint_config` block supports:
@@ -363,6 +365,12 @@ In addition to the arguments listed above, the following computed attributes are
 - `update` - (Default `60 minutes`) Used for updates to node pools
 - `delete` - (Default `60 minutes`) Used for removing node pools.
 
+<a name="nested_kubelet_config"></a>The `kubelet_config` block supports:
+
+* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
+
+* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs.
+
 ## Import
 
 Node pools can be imported using the `project`, `location`, `cluster` and `name`. If