@@ -3,6 +3,7 @@ package infra
33import (
44 "context"
55 "encoding/base64"
6+ "encoding/json"
67 "errors"
78 "fmt"
89 "net/http"
@@ -15,6 +16,7 @@ import (
1516
1617 "github.com/gruntwork-io/terratest/modules/k8s"
1718 "github.com/gruntwork-io/terratest/modules/random"
19+ "github.com/gruntwork-io/terratest/modules/retry"
1820 "github.com/stretchr/testify/require"
1921 cloudkms "google.golang.org/api/cloudkms/v1"
2022 "google.golang.org/api/compute/v1"
@@ -52,6 +54,12 @@ const (
5254 gcpKMSCryptoKeyID = "cockroachdb-cmek-key"
5355)
5456
57+ // getNodeServiceAccount returns the GKE node service account from the environment.
58+ // If empty, gcloud falls back to the project's default Compute Engine service account.
59+ func getNodeServiceAccount () string {
60+ return os .Getenv ("GCP_NODE_SERVICE_ACCOUNT" )
61+ }
62+
5563// getProjectID returns the GCP project ID from the environment variable or falls back to default.
5664func getProjectID () string {
5765 if projectID := os .Getenv ("GCP_PROJECT_ID" ); projectID != "" {
@@ -236,6 +244,72 @@ func (r *GcpRegion) SetUpInfra(t *testing.T) {
236244 require .NoError (t , err )
237245 err = r .deployAndConfigureCoreDNS (t , kubeConfigPath )
238246 require .NoError (t , err , "failed to deploy and configure CoreDNS" )
247+
248+ if r .IsMultiRegion {
249+ for _ , clusterName := range r .Clusters {
250+ err = patchKubeDNSForCustomDomains (t , clusterName , kubeConfigPath )
251+ require .NoError (t , err , "failed to patch kube-dns for custom domains on cluster %s" , clusterName )
252+ }
253+ }
254+ }
255+
256+ // patchKubeDNSForCustomDomains patches the kube-dns ConfigMap with stubDomains for
257+ // custom cluster domains and restarts node-local-dns to pick up the new config.
258+ func patchKubeDNSForCustomDomains (t * testing.T , clusterName , kubeConfigPath string ) error {
259+ kubectlOpts := k8s .NewKubectlOptions (clusterName , kubeConfigPath , "kube-system" )
260+
261+ clusterIP , err := k8s .RunKubectlAndGetOutputE (t , kubectlOpts ,
262+ "get" , "service" , "kube-dns-upstream" , "-o" , "jsonpath={.spec.clusterIP}" )
263+ if err != nil {
264+ return fmt .Errorf ("failed to get kube-dns-upstream ClusterIP on cluster %s: %w" , clusterName , err )
265+ }
266+ clusterIP = strings .TrimSpace (clusterIP )
267+ if clusterIP == "" {
268+ return fmt .Errorf ("kube-dns-upstream service has no ClusterIP on cluster %s" , clusterName )
269+ }
270+ t .Logf ("[gcp] kube-dns-upstream ClusterIP on cluster %s: %s" , clusterName , clusterIP )
271+
272+ stubDomains := make (map [string ][]string )
273+ for _ , domain := range operator .CustomDomains {
274+ stubDomains [domain ] = []string {clusterIP }
275+ }
276+ stubDomainsJSON , err := json .Marshal (stubDomains )
277+ if err != nil {
278+ return fmt .Errorf ("failed to marshal stubDomains: %w" , err )
279+ }
280+
281+ type configMapPatch struct {
282+ Data map [string ]string `json:"data"`
283+ }
284+ patchJSON , err := json .Marshal (configMapPatch {Data : map [string ]string {
285+ "stubDomains" : string (stubDomainsJSON ),
286+ }})
287+ if err != nil {
288+ return fmt .Errorf ("failed to marshal ConfigMap patch: %w" , err )
289+ }
290+
291+ if err := k8s .RunKubectlE (t , kubectlOpts , "patch" , "configmap" , "kube-dns" ,
292+ "--type=merge" , "-p" , string (patchJSON )); err != nil {
293+ return fmt .Errorf ("failed to patch kube-dns ConfigMap on cluster %s: %w" , clusterName , err )
294+ }
295+ t .Logf ("[gcp] Patched kube-dns ConfigMap with stubDomains on cluster %s" , clusterName )
296+
297+ if err := k8s .RunKubectlE (t , kubectlOpts , "delete" , "pods" ,
298+ "-l" , "k8s-app=node-local-dns" , "--grace-period=0" , "--force" ); err != nil {
299+ t .Logf ("[gcp] Warning: force-delete of node-local-dns pods failed (may be harmless): %v" , err )
300+ }
301+
302+ _ , err = retry .DoWithRetryE (t , "wait for node-local-dns rollout" , defaultRetries , defaultRetryInterval ,
303+ func () (string , error ) {
304+ return k8s .RunKubectlAndGetOutputE (t , kubectlOpts ,
305+ "rollout" , "status" , "daemonset/node-local-dns" , "--timeout=10s" )
306+ })
307+ if err != nil {
308+ return fmt .Errorf ("node-local-dns did not become ready after patching on cluster %s: %w" , clusterName , err )
309+ }
310+
311+ t .Logf ("[gcp] node-local-dns ready with stub domains for custom cluster domains on cluster %s" , clusterName )
312+ return nil
239313}
240314
241315func (r * GcpRegion ) GetEncryptionProvider () encryption.Provider {
@@ -787,16 +861,20 @@ func createGKERegionalCluster(ctx context.Context, client *container.Service, se
787861 "--tags" , strings .Join ([]string {defaultNodeTag }, "," ), // Join tags if there are multiple
788862 "--enable-master-authorized-networks" ,
789863 "--master-authorized-networks" , strings .Join ([]string {"0.0.0.0/0" }, "," ),
790- "--num-nodes" , fmt .Sprint (defaultNodesPerZone ),
864+ "--num-nodes" , fmt .Sprint (defaultNodesPerZone + 1 ), // 2 nodes/zone avoids autoscaling during TestClusterScaleUp
791865 "--min-nodes" , fmt .Sprint (defaultNodesPerZone ),
792- "--max-nodes" , fmt .Sprint (defaultNodesPerZone + 1 ), // Needed for scaling cluster
866+ "--max-nodes" , fmt .Sprint (defaultNodesPerZone + 2 ), // headroom above initial count
793867 "--enable-autoscaling" , // Enable autoscaling
794868 "--autoprovisioning-network-tags" , strings .Join ([]string {autoprovisioningNodeTag }, "," ),
795869 "--machine-type" , gcpDefaultMachineType ,
796870 "--disk-size" , "30GB" , // Limit disk size to 30GB
797871 "--quiet" , // Suppress interactive prompts
798872 }
799873
874+ if sa := getNodeServiceAccount (); sa != "" {
875+ args = append (args , "--service-account" , sa )
876+ }
877+
800878 cmd := exec .Command ("gcloud" , args ... )
801879
802880 // Stream gcloud's stdout/stderr directly for real-time visibility into the long-running creation process.
0 commit comments