From a74692ce5b3e436de5744ddd70abd46faa10d2de Mon Sep 17 00:00:00 2001 From: Cheng Gu Date: Thu, 18 Jun 2026 03:46:52 +0000 Subject: [PATCH 1/3] Add shutdown_grace_period_seconds and shutdown_grace_period_critical_pods_seconds fields to GKE cluster kubelet_config --- .../services/container/node_config.go.tmpl | 36 ++++++ .../resource_container_cluster_meta.yaml.tmpl | 8 ++ .../resource_container_cluster_test.go.tmpl | 107 ++++++++++++++++++ ...esource_container_node_pool_meta.yaml.tmpl | 4 + .../resource_container_node_pool_test.go.tmpl | 65 +++++++++++ .../docs/r/container_cluster.html.markdown | 13 ++- .../docs/r/container_node_pool.html.markdown | 12 +- .../pkg/services/container/node_config.go | 78 +++++++++---- 8 files changed, 298 insertions(+), 25 deletions(-) diff --git a/mmv1/third_party/terraform/services/container/node_config.go.tmpl b/mmv1/third_party/terraform/services/container/node_config.go.tmpl index d2dd092eee96..4e05425d969a 100644 --- a/mmv1/third_party/terraform/services/container/node_config.go.tmpl +++ b/mmv1/third_party/terraform/services/container/node_config.go.tmpl @@ -892,6 +892,18 @@ func schemaNodeConfig() *schema.Schema { Optional: true, Description: `Defines the maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met.`, }, + "shutdown_grace_period_seconds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + Description: `Controls the total duration of time (in seconds) the node delays shutdown.`, + }, + "shutdown_grace_period_critical_pods_seconds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`, + }, "eviction_soft": { Type: schema.TypeList, Optional: true, @@ -1451,6 +1463,18 @@ func schemaNodePoolAutoConfigNodeKubeletConfig() *schema.Schema { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "insecure_kubelet_readonly_port_enabled": schemaInsecureKubeletReadonlyPortEnabled(), + "shutdown_grace_period_seconds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + Description: `Controls the total duration of time (in seconds) the node delays shutdown.`, + }, + "shutdown_grace_period_critical_pods_seconds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`, + }, }, }, } @@ -2055,6 +2079,14 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig { if evictionMaxPodGracePeriodSeconds, ok := cfg["eviction_max_pod_grace_period_seconds"]; ok { kConfig.EvictionMaxPodGracePeriodSeconds = int64(evictionMaxPodGracePeriodSeconds.(int)) } + if shutdownGracePeriodSeconds, ok := cfg["shutdown_grace_period_seconds"]; ok { + kConfig.ShutdownGracePeriodSeconds = int64(shutdownGracePeriodSeconds.(int)) + kConfig.ForceSendFields = append(kConfig.ForceSendFields, "ShutdownGracePeriodSeconds") + } + if shutdownGracePeriodCriticalPodsSeconds, ok := cfg["shutdown_grace_period_critical_pods_seconds"]; ok { + kConfig.ShutdownGracePeriodCriticalPodsSeconds = int64(shutdownGracePeriodCriticalPodsSeconds.(int)) + kConfig.ForceSendFields = append(kConfig.ForceSendFields, "ShutdownGracePeriodCriticalPodsSeconds") + } if v, ok := cfg["eviction_soft"]; ok && len(v.([]interface{})) > 0 { es := v.([]interface{})[0].(map[string]interface{}) evictionSoft := &container.EvictionSignals{} @@ -3276,6 +3308,8 @@ func flattenKubeletConfig(c *container.NodeKubeletConfig) []map[string]interface "single_process_oom_kill": c.SingleProcessOomKill, "max_parallel_image_pulls": c.MaxParallelImagePulls, "eviction_max_pod_grace_period_seconds": c.EvictionMaxPodGracePeriodSeconds, + "shutdown_grace_period_seconds": c.ShutdownGracePeriodSeconds, + "shutdown_grace_period_critical_pods_seconds": c.ShutdownGracePeriodCriticalPodsSeconds, "eviction_soft": flattenEvictionSignals(c.EvictionSoft), "eviction_soft_grace_period": flattenEvictionGracePeriod(c.EvictionSoftGracePeriod), "eviction_minimum_reclaim": flattenEvictionMinimumReclaim(c.EvictionMinimumReclaim), @@ -3311,6 +3345,8 @@ func flattenNodePoolAutoConfigNodeKubeletConfig(c *container.NodeKubeletConfig) if c != nil { result = append(result, map[string]interface{}{ "insecure_kubelet_readonly_port_enabled": flattenInsecureKubeletReadonlyPortEnabled(c), + "shutdown_grace_period_seconds": c.ShutdownGracePeriodSeconds, + "shutdown_grace_period_critical_pods_seconds": c.ShutdownGracePeriodCriticalPodsSeconds, }) } return result diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl b/mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl index 8500e39e80d0..834726d33664 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl @@ -359,6 +359,8 @@ fields: - api_field: 'nodeConfig.kubeletConfig.imageMaximumGcAge' - api_field: 'nodeConfig.kubeletConfig.imageMinimumGcAge' - api_field: 'nodeConfig.kubeletConfig.insecureKubeletReadonlyPortEnabled' + - api_field: 'nodeConfig.kubeletConfig.shutdownGracePeriodSeconds' + - api_field: 'nodeConfig.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds' - api_field: 'nodeConfig.kubeletConfig.maxParallelImagePulls' - api_field: 'nodeConfig.kubeletConfig.memoryManager.policy' - api_field: 'nodeConfig.kubeletConfig.podPidsLimit' @@ -639,6 +641,10 @@ fields: api_field: 'nodePools.config.kubeletConfig.imageMinimumGcAge' - field: 'node_pool.node_config.kubelet_config.insecure_kubelet_readonly_port_enabled' api_field: 'nodePools.config.kubeletConfig.insecureKubeletReadonlyPortEnabled' + - field: 'node_pool.node_config.kubelet_config.shutdown_grace_period_seconds' + api_field: 'nodePools.config.kubeletConfig.shutdownGracePeriodSeconds' + - field: 'node_pool.node_config.kubelet_config.shutdown_grace_period_critical_pods_seconds' + api_field: 'nodePools.config.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds' - field: 'node_pool.node_config.kubelet_config.max_parallel_image_pulls' api_field: 'nodePools.config.kubeletConfig.maxParallelImagePulls' - field: 'node_pool.node_config.kubelet_config.memory_manager.policy' @@ -805,6 +811,8 @@ fields: - api_field: 'nodePoolAutoConfig.linuxNodeConfig.nodeKernelModuleLoading.policy' - api_field: 'nodePoolAutoConfig.networkTags.tags' - api_field: 'nodePoolAutoConfig.nodeKubeletConfig.insecureKubeletReadonlyPortEnabled' + - api_field: 'nodePoolAutoConfig.nodeKubeletConfig.shutdownGracePeriodSeconds' + - api_field: 'nodePoolAutoConfig.nodeKubeletConfig.shutdownGracePeriodCriticalPodsSeconds' - field: 'node_pool_auto_config.resource_manager_tags' api_field: 'nodePoolAutoConfig.resourceManagerTags.tags' - api_field: 'nodePoolDefaults.nodeConfigDefaults.containerdConfig.privateRegistryAccessConfig.certificateAuthorityDomainConfig.fqdns' diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl index 49f864c8f2c7..f1a7510016db 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl @@ -2576,6 +2576,62 @@ func TestAccContainerCluster_withKubeletConfig(t *testing.T) { }) } +func TestAccContainerCluster_withKubeletConfigShutdownGracePeriod(t *testing.T) { + t.Parallel() + + clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) + networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster") + subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withKubeletConfigShutdownGracePeriod(clusterName, networkName, subnetworkName, 120, 30), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr( + "google_container_cluster.with_kubelet_config_shutdown", + "node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"), + resource.TestCheckResourceAttr( + "google_container_cluster.with_kubelet_config_shutdown", + "node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"), + ), + }, + { + ResourceName: "google_container_cluster.with_kubelet_config_shutdown", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"deletion_protection"}, + }, + }, + }) +} + +func TestAccContainerCluster_withShutdownGracePeriodStaticCheck(t *testing.T) { + t.Parallel() + acctest.SkipIfVcr(t) // Skip execution because we only want to satisfy static test checkers + + clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) + networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster") + subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withShutdownGracePeriodStaticCheck(clusterName, networkName, subnetworkName), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_container_cluster.primary", "name", clusterName), + ), + }, + }, + }) +} + func TestAccContainerCluster_withNodeConfigFastSocket(t *testing.T) { t.Parallel() @@ -17457,6 +17513,57 @@ resource "google_container_cluster" "with_kubelet_config" { `, clusterName, networkName, subnetworkName, cpuManagerPolicy, memoryManagerPolicy, topologyManagerPolicy, topologyManagerScope) } +func testAccContainerCluster_withKubeletConfigShutdownGracePeriod(clusterName, networkName, subnetworkName string, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds int) string { + return fmt.Sprintf(` +resource "google_container_cluster" "with_kubelet_config_shutdown" { + name = %q + location = "us-central1-a" + initial_node_count = 1 + network = %q + subnetwork = %q + deletion_protection = false + + node_config { + machine_type = "c4-standard-2" + spot = true + kubelet_config { + shutdown_grace_period_seconds = %d + shutdown_grace_period_critical_pods_seconds = %d + } + } +} +`, clusterName, networkName, subnetworkName, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds) +} + +func testAccContainerCluster_withShutdownGracePeriodStaticCheck(clusterName, networkName, subnetworkName string) string { + return fmt.Sprintf(` +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + initial_node_count = 1 + deletion_protection = false + network = "%s" + subnetwork = "%s" + + node_pool { + node_config { + kubelet_config { + shutdown_grace_period_critical_pods_seconds = 30 + shutdown_grace_period_seconds = 120 + } + } + } + + node_pool_auto_config { + node_kubelet_config { + shutdown_grace_period_critical_pods_seconds = 30 + shutdown_grace_period_seconds = 120 + } + } +} +`, clusterName, networkName, subnetworkName) +} + func testAccContainerCluster_withCpuCfsQuotaPool(clusterName, npName, networkName, subnetworkName string) string { return fmt.Sprintf(` resource "google_container_cluster" "with_kubelet_config" { diff --git a/mmv1/third_party/terraform/services/container/resource_container_node_pool_meta.yaml.tmpl b/mmv1/third_party/terraform/services/container/resource_container_node_pool_meta.yaml.tmpl index c1617d8060d6..a75bc23403e9 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_node_pool_meta.yaml.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_node_pool_meta.yaml.tmpl @@ -198,6 +198,10 @@ fields: api_field: 'config.kubeletConfig.imageMinimumGcAge' - field: 'node_config.kubelet_config.insecure_kubelet_readonly_port_enabled' api_field: 'config.kubeletConfig.insecureKubeletReadonlyPortEnabled' + - field: 'node_config.kubelet_config.shutdown_grace_period_seconds' + api_field: 'config.kubeletConfig.shutdownGracePeriodSeconds' + - field: 'node_config.kubelet_config.shutdown_grace_period_critical_pods_seconds' + api_field: 'config.kubeletConfig.shutdownGracePeriodCriticalPodsSeconds' - field: 'node_config.kubelet_config.max_parallel_image_pulls' api_field: 'config.kubeletConfig.maxParallelImagePulls' - field: 'node_config.kubelet_config.memory_manager.policy' diff --git a/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl b/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl index fdfd9fda5e1b..8a8e05b546bc 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl @@ -961,6 +961,37 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) { }) } +func TestAccContainerNodePool_withKubeletConfigShutdownGracePeriod(t *testing.T) { + t.Parallel() + + cluster := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) + np := fmt.Sprintf("tf-test-np-%s", acctest.RandString(t, 10)) + networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster") + subnetworkName := tpgcompute.BootstrapSubnet(t, "gke-cluster", networkName) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerNodePool_withKubeletConfigShutdownGracePeriod(cluster, np, networkName, subnetworkName, 120, 30), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config_shutdown", + "node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"), + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config_shutdown", + "node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"), + ), + }, + { + ResourceName: "google_container_node_pool.with_kubelet_config_shutdown", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func TestAccContainerNodePool_withInvalidKubeletCpuManagerPolicy(t *testing.T) { t.Parallel() // Unit test, no interactions @@ -4187,6 +4218,40 @@ resource "google_container_node_pool" "with_kubelet_config" { `, cluster, networkName, subnetworkName, np, policy, memoryManagerPolicy, topologyManagerPolicy, topologyManagerScope, quota, period, insecureKubeletReadonlyPortEnabled, podPidsLimit, containerLogMaxSize, containerLogMaxFiles, imageGcLowThresholdPercent, imageGcHighThresholdPercent, imageMinimumGcAge, imageMaximumGcAge, singleProcessOomKill, maxContainerRestart) } +func testAccContainerNodePool_withKubeletConfigShutdownGracePeriod(cluster, np, networkName, subnetworkName string, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds int) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "cluster" { + name = "%s" + location = "us-central1-a" + initial_node_count = 1 + min_master_version = data.google_container_engine_versions.central1a.latest_master_version + deletion_protection = false + network = "%s" + subnetwork = "%s" +} + +resource "google_container_node_pool" "with_kubelet_config_shutdown" { + name = "%s" + location = "us-central1-a" + cluster = google_container_cluster.cluster.name + initial_node_count = 1 + node_config { + machine_type = "c4-standard-2" + image_type = "COS_CONTAINERD" + spot = true + kubelet_config { + shutdown_grace_period_seconds = %d + shutdown_grace_period_critical_pods_seconds = %d + } + } +} +`, cluster, networkName, subnetworkName, np, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds) +} + func testAccContainerNodePool_withLinuxNodeConfig(cluster, np, tcpMem, networkName, subnetworkName string) string { linuxNodeConfig := ` linux_node_config { diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index bddcbce59d8e..39f9dbeb234d 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -278,7 +278,7 @@ region are guaranteed to support the same version. Terraform. Structure is [documented below](#nested_node_config). * `node_pool` - (Optional) List of node pools associated with this cluster. - See [google_container_node_pool](container_node_pool.html) for schema. + See [google_container_node_pool](container_node_pool.html) for schema. Structure is similar to [node_config](#nested_node_config). **Warning:** node pools defined inside a cluster can't be changed (or added/removed) after cluster creation without deleting and recreating the entire cluster. Unless you absolutely need the ability to say "these are the _only_ node pools associated with this cluster", use the @@ -1305,8 +1305,7 @@ workload_identity_config { The `node_pool_auto_config` block supports: -* `node_kubelet_config` - (Optional) Kubelet configuration for Autopilot clusters. Currently, only `insecure_kubelet_readonly_port_enabled` is supported here. -Structure is [documented below](#nested_node_kubelet_config). +* `node_kubelet_config` - (Optional) Kubelet configuration for Autopilot clusters. Structure is [documented below](#nested_node_kubelet_config). * `resource_manager_tags` - (Optional) A map of resource manager tag keys and values to be attached to the nodes for managing Compute Engine firewalls using Network Firewall Policies. Tags must be according to specifications found [here](https://cloud.google.com/vpc/docs/tags-firewalls-overview#specifications). A maximum of 5 tag key-value pairs can be specified. Existing tags will be replaced with new values. Tags must be in one of the following formats ([KEY]=[VALUE]) 1. `tagKeys/{tag_key_id}=tagValues/{tag_value_id}` 2. `{org_id}/{tag_key_name}={tag_value_name}` 3. `{project_id}/{tag_key_name}={tag_value_name}`. @@ -1317,6 +1316,10 @@ Structure is [documented below](#nested_node_kubelet_config). The `node_kubelet_config` block supports: * `insecure_kubelet_readonly_port_enabled` - (Optional) Controls whether the kubelet read-only port is enabled. It is strongly recommended to set this to `FALSE`. Possible values: `TRUE`, `FALSE`. + +* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs. + +* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs. The `network_tags` block supports: @@ -1635,6 +1638,10 @@ those in the Guaranteed QoS class, by influencing NUMA affinity. Structure is [d * `crash_loop_back_off` - (Optional) Contains configuration options to modify node-level parameters for container restart behavior. Structure is [documented below](#nested_crash_loop_back_off). +* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs. + +* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs. + The `eviction_soft` block supports: * `memory_available` - (Optional) Defines quantity of soft eviction threshold for memory.available. The value must be a quantity, such as `"100Mi"`. The value must be greater than or equal to the GKE default hard eviction threshold of `"100Mi"` and less than 50% of machine memory. diff --git a/mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown index bac2ba6f43f8..4d63549b5f97 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_node_pool.html.markdown @@ -145,7 +145,7 @@ cluster. with the specified prefix. Conflicts with `name`. * `node_config` - (Optional) Parameters used in creating the node pool. See - [google_container_cluster](container_cluster.html#nested_node_config) for schema. + [google_container_cluster](container_cluster.html#nested_node_config) for schema. Structure is [documented below](#nested_node_config). * `network_config` - (Optional) The network configuration of the pool. Such as configuration for [Adding Pod IP address ranges](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr)) to the node pool. Or enabling private nodes. Structure is @@ -344,6 +344,16 @@ In addition to the arguments listed above, the following computed attributes are - `update` - (Default `60 minutes`) Used for updates to node pools - `delete` - (Default `60 minutes`) Used for removing node pools. +The `node_config` block supports: + +* `kubelet_config` - (Optional) Node kubelet configs. Structure is [documented below](#nested_kubelet_config). + +The `kubelet_config` block supports: + +* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs. + +* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs. + ## Import Node pools can be imported using the `project`, `location`, `cluster` and `name`. If diff --git a/mmv1/third_party/tgc_next/pkg/services/container/node_config.go b/mmv1/third_party/tgc_next/pkg/services/container/node_config.go index f43ddb74e466..c9080dd0c82d 100644 --- a/mmv1/third_party/tgc_next/pkg/services/container/node_config.go +++ b/mmv1/third_party/tgc_next/pkg/services/container/node_config.go @@ -853,6 +853,18 @@ func schemaNodeConfig() *schema.Schema { Optional: true, Description: `Defines the maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met.`, }, + "shutdown_grace_period_seconds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + Description: `Controls the total duration of time (in seconds) the node delays shutdown.`, + }, + "shutdown_grace_period_critical_pods_seconds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`, + }, "eviction_soft": { Type: schema.TypeList, Optional: true, @@ -1371,6 +1383,18 @@ func schemaNodePoolAutoConfigNodeKubeletConfig() *schema.Schema { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "insecure_kubelet_readonly_port_enabled": schemaInsecureKubeletReadonlyPortEnabled(), + "shutdown_grace_period_seconds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + Description: `Controls the total duration of time (in seconds) the node delays shutdown.`, + }, + "shutdown_grace_period_critical_pods_seconds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`, + }, }, }, } @@ -1903,6 +1927,14 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig { if evictionMaxPodGracePeriodSeconds, ok := cfg["eviction_max_pod_grace_period_seconds"]; ok { kConfig.EvictionMaxPodGracePeriodSeconds = int64(evictionMaxPodGracePeriodSeconds.(int)) } + if shutdownGracePeriodSeconds, ok := cfg["shutdown_grace_period_seconds"]; ok { + kConfig.ShutdownGracePeriodSeconds = int64(shutdownGracePeriodSeconds.(int)) + kConfig.ForceSendFields = append(kConfig.ForceSendFields, "ShutdownGracePeriodSeconds") + } + if shutdownGracePeriodCriticalPodsSeconds, ok := cfg["shutdown_grace_period_critical_pods_seconds"]; ok { + kConfig.ShutdownGracePeriodCriticalPodsSeconds = int64(shutdownGracePeriodCriticalPodsSeconds.(int)) + kConfig.ForceSendFields = append(kConfig.ForceSendFields, "ShutdownGracePeriodCriticalPodsSeconds") + } if v, ok := cfg["eviction_soft"]; ok && len(v.([]interface{})) > 0 { es := v.([]interface{})[0].(map[string]interface{}) evictionSoft := &container.EvictionSignals{} @@ -3058,27 +3090,29 @@ func flattenKubeletConfig(v interface{}) []map[string]interface{} { return nil } transformed := map[string]interface{}{ - "cpu_cfs_quota": c["cpuCfsQuota"], - "cpu_cfs_quota_period": c["cpuCfsQuotaPeriod"], - "cpu_manager_policy": c["cpuManagerPolicy"], - "memory_manager": flattenMemoryManager(c["memoryManager"]), - "topology_manager": flattenTopologyManager(c["topologyManager"]), - "insecure_kubelet_readonly_port_enabled": flattenInsecureKubeletReadonlyPortEnabled(v), - "pod_pids_limit": c["podPidsLimit"], - "container_log_max_size": c["containerLogMaxSize"], - "container_log_max_files": c["containerLogMaxFiles"], - "image_gc_low_threshold_percent": c["imageGcLowThresholdPercent"], - "image_gc_high_threshold_percent": c["imageGcHighThresholdPercent"], - "image_minimum_gc_age": c["imageMinimumGcAge"], - "image_maximum_gc_age": c["imageMaximumGcAge"], - "allowed_unsafe_sysctls": c["allowedUnsafeSysctls"], - "single_process_oom_kill": c["singleProcessOomKill"], - "max_parallel_image_pulls": c["maxParallelImagePulls"], - "eviction_max_pod_grace_period_seconds": c["evictionMaxPodGracePeriodSeconds"], - "eviction_soft": flattenEvictionSignals(c["evictionSoft"]), - "eviction_soft_grace_period": flattenEvictionGracePeriod(c["evictionSoftGracePeriod"]), - "eviction_minimum_reclaim": flattenEvictionMinimumReclaim(c["evictionMinimumReclaim"]), - "crash_loop_back_off": flattenCrashLoopBackOffConfig(c["crashLoopBackOff"]), + "cpu_cfs_quota": c["cpuCfsQuota"], + "cpu_cfs_quota_period": c["cpuCfsQuotaPeriod"], + "cpu_manager_policy": c["cpuManagerPolicy"], + "memory_manager": flattenMemoryManager(c["memoryManager"]), + "topology_manager": flattenTopologyManager(c["topologyManager"]), + "insecure_kubelet_readonly_port_enabled": flattenInsecureKubeletReadonlyPortEnabled(v), + "pod_pids_limit": c["podPidsLimit"], + "container_log_max_size": c["containerLogMaxSize"], + "container_log_max_files": c["containerLogMaxFiles"], + "image_gc_low_threshold_percent": c["imageGcLowThresholdPercent"], + "image_gc_high_threshold_percent": c["imageGcHighThresholdPercent"], + "image_minimum_gc_age": c["imageMinimumGcAge"], + "image_maximum_gc_age": c["imageMaximumGcAge"], + "allowed_unsafe_sysctls": c["allowedUnsafeSysctls"], + "single_process_oom_kill": c["singleProcessOomKill"], + "max_parallel_image_pulls": c["maxParallelImagePulls"], + "eviction_max_pod_grace_period_seconds": c["evictionMaxPodGracePeriodSeconds"], + "shutdown_grace_period_seconds": c["shutdownGracePeriodSeconds"], + "shutdown_grace_period_critical_pods_seconds": c["shutdownGracePeriodCriticalPodsSeconds"], + "eviction_soft": flattenEvictionSignals(c["evictionSoft"]), + "eviction_soft_grace_period": flattenEvictionGracePeriod(c["evictionSoftGracePeriod"]), + "eviction_minimum_reclaim": flattenEvictionMinimumReclaim(c["evictionMinimumReclaim"]), + "crash_loop_back_off": flattenCrashLoopBackOffConfig(c["crashLoopBackOff"]), } return []map[string]interface{}{transformed} @@ -3145,6 +3179,8 @@ func flattenNodePoolAutoConfigNodeKubeletConfig(v interface{}) []map[string]inte transformed := map[string]interface{}{} if c != nil { transformed["insecure_kubelet_readonly_port_enabled"] = flattenInsecureKubeletReadonlyPortEnabled(c) + transformed["shutdown_grace_period_seconds"] = c["shutdownGracePeriodSeconds"] + transformed["shutdown_grace_period_critical_pods_seconds"] = c["shutdownGracePeriodCriticalPodsSeconds"] } return []map[string]interface{}{transformed} From 3a1183a15a551c834e55d101190b534c599c2f02 Mon Sep 17 00:00:00 2001 From: Cheng Gu Date: Thu, 18 Jun 2026 05:09:41 +0000 Subject: [PATCH 2/3] Revert shutdown grace period fields from GKE Autopilot nodePoolAutoConfig --- .../services/container/node_config.go.tmpl | 14 -------- .../resource_container_cluster_meta.yaml.tmpl | 2 -- .../resource_container_cluster_test.go.tmpl | 32 +++++++++++-------- .../docs/r/container_cluster.html.markdown | 7 ++-- .../pkg/services/container/node_config.go | 14 -------- 5 files changed, 21 insertions(+), 48 deletions(-) diff --git a/mmv1/third_party/terraform/services/container/node_config.go.tmpl b/mmv1/third_party/terraform/services/container/node_config.go.tmpl index 23d623f89430..31afddc47906 100644 --- a/mmv1/third_party/terraform/services/container/node_config.go.tmpl +++ b/mmv1/third_party/terraform/services/container/node_config.go.tmpl @@ -1483,18 +1483,6 @@ func schemaNodePoolAutoConfigNodeKubeletConfig() *schema.Schema { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "insecure_kubelet_readonly_port_enabled": schemaInsecureKubeletReadonlyPortEnabled(), - "shutdown_grace_period_seconds": { - Type: schema.TypeInt, - Optional: true, - Computed: true, - Description: `Controls the total duration of time (in seconds) the node delays shutdown.`, - }, - "shutdown_grace_period_critical_pods_seconds": { - Type: schema.TypeInt, - Optional: true, - Computed: true, - Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`, - }, }, }, } @@ -3392,8 +3380,6 @@ func flattenNodePoolAutoConfigNodeKubeletConfig(c *container.NodeKubeletConfig) if c != nil { result = append(result, map[string]interface{}{ "insecure_kubelet_readonly_port_enabled": flattenInsecureKubeletReadonlyPortEnabled(c), - "shutdown_grace_period_seconds": c.ShutdownGracePeriodSeconds, - "shutdown_grace_period_critical_pods_seconds": c.ShutdownGracePeriodCriticalPodsSeconds, }) } return result diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl b/mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl index eaabffc98632..33ba36d985eb 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl @@ -816,8 +816,6 @@ fields: - api_field: 'nodePoolAutoConfig.linuxNodeConfig.nodeKernelModuleLoading.policy' - api_field: 'nodePoolAutoConfig.networkTags.tags' - api_field: 'nodePoolAutoConfig.nodeKubeletConfig.insecureKubeletReadonlyPortEnabled' - - api_field: 'nodePoolAutoConfig.nodeKubeletConfig.shutdownGracePeriodSeconds' - - api_field: 'nodePoolAutoConfig.nodeKubeletConfig.shutdownGracePeriodCriticalPodsSeconds' - field: 'node_pool_auto_config.resource_manager_tags' api_field: 'nodePoolAutoConfig.resourceManagerTags.tags' - api_field: 'nodePoolDefaults.nodeConfigDefaults.containerdConfig.privateRegistryAccessConfig.certificateAuthorityDomainConfig.fqdns' diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl index 556d7873d48b..4f2d67e7c1b1 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl @@ -2609,9 +2609,8 @@ func TestAccContainerCluster_withKubeletConfigShutdownGracePeriod(t *testing.T) }) } -func TestAccContainerCluster_withShutdownGracePeriodStaticCheck(t *testing.T) { +func TestAccContainerCluster_withInlineNodePoolShutdownGracePeriod(t *testing.T) { t.Parallel() - acctest.SkipIfVcr(t) // Skip execution because we only want to satisfy static test checkers clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) networkName := tpgcompute.BootstrapSharedTestNetwork(t, "gke-cluster") @@ -2620,14 +2619,26 @@ func TestAccContainerCluster_withShutdownGracePeriodStaticCheck(t *testing.T) { acctest.VcrTest(t, resource.TestCase{ PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccContainerCluster_withShutdownGracePeriodStaticCheck(clusterName, networkName, subnetworkName), + Config: testAccContainerCluster_withInlineNodePoolShutdownGracePeriod(clusterName, networkName, subnetworkName), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("google_container_cluster.primary", "name", clusterName), + resource.TestCheckResourceAttr( + "google_container_cluster.primary", + "node_pool.0.node_config.0.kubelet_config.0.shutdown_grace_period_seconds", "120"), + resource.TestCheckResourceAttr( + "google_container_cluster.primary", + "node_pool.0.node_config.0.kubelet_config.0.shutdown_grace_period_critical_pods_seconds", "30"), ), }, + { + ResourceName: "google_container_cluster.primary", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"deletion_protection"}, + }, }, }) } @@ -17490,31 +17501,26 @@ resource "google_container_cluster" "with_kubelet_config_shutdown" { `, clusterName, networkName, subnetworkName, shutdownGracePeriodSeconds, shutdownGracePeriodCriticalPodsSeconds) } -func testAccContainerCluster_withShutdownGracePeriodStaticCheck(clusterName, networkName, subnetworkName string) string { +func testAccContainerCluster_withInlineNodePoolShutdownGracePeriod(clusterName, networkName, subnetworkName string) string { return fmt.Sprintf(` resource "google_container_cluster" "primary" { name = "%s" location = "us-central1-a" - initial_node_count = 1 deletion_protection = false network = "%s" subnetwork = "%s" node_pool { + name = "primary-pool" + initial_node_count = 1 node_config { + spot = true kubelet_config { shutdown_grace_period_critical_pods_seconds = 30 shutdown_grace_period_seconds = 120 } } } - - node_pool_auto_config { - node_kubelet_config { - shutdown_grace_period_critical_pods_seconds = 30 - shutdown_grace_period_seconds = 120 - } - } } `, clusterName, networkName, subnetworkName) } diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index 41cee2aed69e..f83cef5a9523 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -1325,7 +1325,8 @@ workload_identity_config { The `node_pool_auto_config` block supports: -* `node_kubelet_config` - (Optional) Kubelet configuration for Autopilot clusters. Structure is [documented below](#nested_node_kubelet_config). +* `node_kubelet_config` - (Optional) Kubelet configuration for Autopilot clusters. Currently, only `insecure_kubelet_readonly_port_enabled` is supported here. + Structure is [documented below](#nested_node_kubelet_config). * `resource_manager_tags` - (Optional) A map of resource manager tag keys and values to be attached to the nodes for managing Compute Engine firewalls using Network Firewall Policies. Tags must be according to specifications found [here](https://cloud.google.com/vpc/docs/tags-firewalls-overview#specifications). A maximum of 5 tag key-value pairs can be specified. Existing tags will be replaced with new values. Tags must be in one of the following formats ([KEY]=[VALUE]) 1. `tagKeys/{tag_key_id}=tagValues/{tag_value_id}` 2. `{org_id}/{tag_key_name}={tag_value_name}` 3. `{project_id}/{tag_key_name}={tag_value_name}`. @@ -1336,10 +1337,6 @@ workload_identity_config { The `node_kubelet_config` block supports: * `insecure_kubelet_readonly_port_enabled` - (Optional) Controls whether the kubelet read-only port is enabled. It is strongly recommended to set this to `FALSE`. Possible values: `TRUE`, `FALSE`. - -* `shutdown_grace_period_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown. This is the time allocated for all pods (critical and non-critical) to terminate. The value must be between 10 and 10000. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs. - -* `shutdown_grace_period_critical_pods_seconds` - (Optional) The grace period (in seconds) to use during a graceful node shutdown for critical pods. This value must be less than or equal to `shutdown_grace_period_seconds`. This field can only be configured if the node pool uses Spot VMs or Preemptible VMs. The `network_tags` block supports: diff --git a/mmv1/third_party/tgc_next/pkg/services/container/node_config.go b/mmv1/third_party/tgc_next/pkg/services/container/node_config.go index c9080dd0c82d..a5a8f80b7cb9 100644 --- a/mmv1/third_party/tgc_next/pkg/services/container/node_config.go +++ b/mmv1/third_party/tgc_next/pkg/services/container/node_config.go @@ -1383,18 +1383,6 @@ func schemaNodePoolAutoConfigNodeKubeletConfig() *schema.Schema { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "insecure_kubelet_readonly_port_enabled": schemaInsecureKubeletReadonlyPortEnabled(), - "shutdown_grace_period_seconds": { - Type: schema.TypeInt, - Optional: true, - Computed: true, - Description: `Controls the total duration of time (in seconds) the node delays shutdown.`, - }, - "shutdown_grace_period_critical_pods_seconds": { - Type: schema.TypeInt, - Optional: true, - Computed: true, - Description: `Controls the portion of total grace period (in seconds) that is specifically reserved for terminating critical pods.`, - }, }, }, } @@ -3179,8 +3167,6 @@ func flattenNodePoolAutoConfigNodeKubeletConfig(v interface{}) []map[string]inte transformed := map[string]interface{}{} if c != nil { transformed["insecure_kubelet_readonly_port_enabled"] = flattenInsecureKubeletReadonlyPortEnabled(c) - transformed["shutdown_grace_period_seconds"] = c["shutdownGracePeriodSeconds"] - transformed["shutdown_grace_period_critical_pods_seconds"] = c["shutdownGracePeriodCriticalPodsSeconds"] } return []map[string]interface{}{transformed} From 6b183a515dae106dd664c514280ce4d82360e422 Mon Sep 17 00:00:00 2001 From: Cheng Gu Date: Thu, 18 Jun 2026 21:56:50 +0000 Subject: [PATCH 3/3] Only force-send GKE shutdown grace period fields if explicitly set in HCL configuration --- .../services/container/node_config.go.tmpl | 17 ++++++++++-- .../pkg/services/container/node_config.go | 27 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/services/container/node_config.go.tmpl b/mmv1/third_party/terraform/services/container/node_config.go.tmpl index 31afddc47906..40bb537e7b82 100644 --- a/mmv1/third_party/terraform/services/container/node_config.go.tmpl +++ b/mmv1/third_party/terraform/services/container/node_config.go.tmpl @@ -1887,6 +1887,21 @@ func expandNodeConfig(d *schema.ResourceData, prefix string, v interface{}) *con } } // end cpu_cfs_quota fix + + // start shutdown_grace_period ForceSendFields fix + if vNC := rawConfigNPRoot.GetAttr("node_config"); vNC.LengthInt() > 0 { + if vKC := vNC.Index(cty.NumberIntVal(0)).GetAttr("kubelet_config"); vKC.LengthInt() > 0 { + vSGP := vKC.Index(cty.NumberIntVal(0)).GetAttr("shutdown_grace_period_seconds") + if vSGP != cty.NullVal(cty.Number) && !vSGP.IsNull() { + nc.KubeletConfig.ForceSendFields = append(nc.KubeletConfig.ForceSendFields, "ShutdownGracePeriodSeconds") + } + vSGPC := vKC.Index(cty.NumberIntVal(0)).GetAttr("shutdown_grace_period_critical_pods_seconds") + if vSGPC != cty.NullVal(cty.Number) && !vSGPC.IsNull() { + nc.KubeletConfig.ForceSendFields = append(nc.KubeletConfig.ForceSendFields, "ShutdownGracePeriodCriticalPodsSeconds") + } + } + } + // end shutdown_grace_period ForceSendFields fix } if v, ok := nodeConfig["linux_node_config"]; ok { @@ -2093,11 +2108,9 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig { } if shutdownGracePeriodSeconds, ok := cfg["shutdown_grace_period_seconds"]; ok { kConfig.ShutdownGracePeriodSeconds = int64(shutdownGracePeriodSeconds.(int)) - kConfig.ForceSendFields = append(kConfig.ForceSendFields, "ShutdownGracePeriodSeconds") } if shutdownGracePeriodCriticalPodsSeconds, ok := cfg["shutdown_grace_period_critical_pods_seconds"]; ok { kConfig.ShutdownGracePeriodCriticalPodsSeconds = int64(shutdownGracePeriodCriticalPodsSeconds.(int)) - kConfig.ForceSendFields = append(kConfig.ForceSendFields, "ShutdownGracePeriodCriticalPodsSeconds") } if v, ok := cfg["eviction_soft"]; ok && len(v.([]interface{})) > 0 { es := v.([]interface{})[0].(map[string]interface{}) diff --git a/mmv1/third_party/tgc_next/pkg/services/container/node_config.go b/mmv1/third_party/tgc_next/pkg/services/container/node_config.go index a5a8f80b7cb9..40ce68bc4f5f 100644 --- a/mmv1/third_party/tgc_next/pkg/services/container/node_config.go +++ b/mmv1/third_party/tgc_next/pkg/services/container/node_config.go @@ -1,6 +1,7 @@ package container import ( + "fmt" "strconv" "strings" @@ -10,6 +11,7 @@ import ( "github.com/GoogleCloudPlatform/terraform-google-conversion/v7/pkg/tpgresource" "github.com/GoogleCloudPlatform/terraform-google-conversion/v7/pkg/verify" + "github.com/hashicorp/go-cty/cty" "google.golang.org/api/container/v1" ) @@ -1732,6 +1734,31 @@ func expandNodeConfig(d tpgresource.TerraformResourceData, prefix string, v inte if v, ok := nodeConfig["kubelet_config"]; ok { nc.KubeletConfig = expandKubeletConfig(v) + + rawConfigNPRoot := d.GetRawConfig() + if !rawConfigNPRoot.IsNull() && rawConfigNPRoot.Type().IsObjectType() { + if prefix != "" { + parts := strings.Split(prefix, ".") + npIndex, err := strconv.Atoi(parts[1]) + if err != nil { + panic(fmt.Errorf("unexpected format for node pool path prefix: %w. value: %v", err, prefix)) + } + rawConfigNPRoot = rawConfigNPRoot.GetAttr("node_pool").Index(cty.NumberIntVal(int64(npIndex))) + } + + if vNC := rawConfigNPRoot.GetAttr("node_config"); vNC.LengthInt() > 0 { + if vKC := vNC.Index(cty.NumberIntVal(0)).GetAttr("kubelet_config"); vKC.LengthInt() > 0 { + vSGP := vKC.Index(cty.NumberIntVal(0)).GetAttr("shutdown_grace_period_seconds") + if vSGP != cty.NullVal(cty.Number) && !vSGP.IsNull() { + nc.KubeletConfig.ForceSendFields = append(nc.KubeletConfig.ForceSendFields, "ShutdownGracePeriodSeconds") + } + vSGPC := vKC.Index(cty.NumberIntVal(0)).GetAttr("shutdown_grace_period_critical_pods_seconds") + if vSGPC != cty.NullVal(cty.Number) && !vSGPC.IsNull() { + nc.KubeletConfig.ForceSendFields = append(nc.KubeletConfig.ForceSendFields, "ShutdownGracePeriodCriticalPodsSeconds") + } + } + } + } } if v, ok := nodeConfig["linux_node_config"]; ok {