e2e/operator: Fix GCP single-region and multi-region test infrastructure

Author: shreyaskm623

tests/e2e/operator/infra/common.go

@@ -302,8 +302,17 @@ func UpdateKubeconfigGCP(t *testing.T, projectID, region, clusterName, alias str
 		return fmt.Errorf("failed to get GCP credentials for cluster %s: %w", clusterName, err)
 	}
 
-	// Step 2: Rename context
+	// Step 2: Skip TLS verification to avoid x509 errors from GKE's internal CA.
 	longContextName := fmt.Sprintf("gke_%s_%s_%s", projectID, region, clusterName)
+	skipTLSCmd := exec.Command("kubectl", "config", "set-cluster", longContextName,
+		"--insecure-skip-tls-verify=true")
+	output, err = skipTLSCmd.CombinedOutput()
+	if err != nil {
+		t.Logf("kubectl set-cluster insecure-skip-tls-verify command failed. Output:\n%s\n", string(output))
+		return fmt.Errorf("failed to set insecure-skip-tls-verify for cluster %s: %w", clusterName, err)
+	}
+
+	// Step 3: Rename context
 	renameCmd := exec.Command("kubectl", "config", "rename-context", longContextName, alias)
 	output, err = renameCmd.CombinedOutput()
 	if err != nil {

tests/e2e/operator/infra/gcp.go

@@ -2,6 +2,7 @@ package infra
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/gruntwork-io/terratest/modules/retry"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/api/compute/v1"
 	"google.golang.org/api/container/v1"
@@ -44,6 +46,12 @@ const (
 // Default project ID to use if not specified in the environment.
 const defaultProjectID = "helm-testing"
 
+// getNodeServiceAccount returns the GKE node service account from the environment.
+// If empty, gcloud falls back to the project's default Compute Engine service account.
+func getNodeServiceAccount() string {
+	return os.Getenv("GCP_NODE_SERVICE_ACCOUNT")
+}
+
 // getProjectID returns the GCP project ID from the environment variable or falls back to default.
 func getProjectID() string {
 	if projectID := os.Getenv("GCP_PROJECT_ID"); projectID != "" {
@@ -227,6 +235,72 @@ func (r *GcpRegion) SetUpInfra(t *testing.T) {
 	require.NoError(t, err)
 	err = r.deployAndConfigureCoreDNS(t, kubeConfigPath)
 	require.NoError(t, err, "failed to deploy and configure CoreDNS")
+
+	if r.IsMultiRegion {
+		for _, clusterName := range r.Clusters {
+			err = patchKubeDNSForCustomDomains(t, clusterName, kubeConfigPath)
+			require.NoError(t, err, "failed to patch kube-dns for custom domains on cluster %s", clusterName)
+		}
+	}
+}
+
+// patchKubeDNSForCustomDomains patches the kube-dns ConfigMap with stubDomains for
+// custom cluster domains and restarts node-local-dns to pick up the new config.
+func patchKubeDNSForCustomDomains(t *testing.T, clusterName, kubeConfigPath string) error {
+	kubectlOpts := k8s.NewKubectlOptions(clusterName, kubeConfigPath, "kube-system")
+
+	clusterIP, err := k8s.RunKubectlAndGetOutputE(t, kubectlOpts,
+		"get", "service", "kube-dns-upstream", "-o", "jsonpath={.spec.clusterIP}")
+	if err != nil {
+		return fmt.Errorf("failed to get kube-dns-upstream ClusterIP on cluster %s: %w", clusterName, err)
+	}
+	clusterIP = strings.TrimSpace(clusterIP)
+	if clusterIP == "" {
+		return fmt.Errorf("kube-dns-upstream service has no ClusterIP on cluster %s", clusterName)
+	}
+	t.Logf("[gcp] kube-dns-upstream ClusterIP on cluster %s: %s", clusterName, clusterIP)
+
+	stubDomains := make(map[string][]string)
+	for _, domain := range operator.CustomDomains {
+		stubDomains[domain] = []string{clusterIP}
+	}
+	stubDomainsJSON, err := json.Marshal(stubDomains)
+	if err != nil {
+		return fmt.Errorf("failed to marshal stubDomains: %w", err)
+	}
+
+	type configMapPatch struct {
+		Data map[string]string `json:"data"`
+	}
+	patchJSON, err := json.Marshal(configMapPatch{Data: map[string]string{
+		"stubDomains": string(stubDomainsJSON),
+	}})
+	if err != nil {
+		return fmt.Errorf("failed to marshal ConfigMap patch: %w", err)
+	}
+
+	if err := k8s.RunKubectlE(t, kubectlOpts, "patch", "configmap", "kube-dns",
+		"--type=merge", "-p", string(patchJSON)); err != nil {
+		return fmt.Errorf("failed to patch kube-dns ConfigMap on cluster %s: %w", clusterName, err)
+	}
+	t.Logf("[gcp] Patched kube-dns ConfigMap with stubDomains on cluster %s", clusterName)
+
+	if err := k8s.RunKubectlE(t, kubectlOpts, "delete", "pods",
+		"-l", "k8s-app=node-local-dns", "--grace-period=0", "--force"); err != nil {
+		t.Logf("[gcp] Warning: force-delete of node-local-dns pods failed (may be harmless): %v", err)
+	}
+
+	_, err = retry.DoWithRetryE(t, "wait for node-local-dns rollout", defaultRetries, defaultRetryInterval,
+		func() (string, error) {
+			return k8s.RunKubectlAndGetOutputE(t, kubectlOpts,
+				"rollout", "status", "daemonset/node-local-dns", "--timeout=10s")
+		})
+	if err != nil {
+		return fmt.Errorf("node-local-dns did not become ready after patching on cluster %s: %w", clusterName, err)
+	}
+
+	t.Logf("[gcp] node-local-dns ready with stub domains for custom cluster domains on cluster %s", clusterName)
+	return nil
 }
 
 // TeardownInfra deletes all GCP resources created by SetUpInfra.
@@ -627,16 +701,20 @@ func createGKERegionalCluster(ctx context.Context, client *container.Service, se
 		"--tags", strings.Join([]string{defaultNodeTag}, ","), // Join tags if there are multiple
 		"--enable-master-authorized-networks",
 		"--master-authorized-networks", strings.Join([]string{"0.0.0.0/0"}, ","),
-		"--num-nodes", fmt.Sprint(defaultNodesPerZone),
+		"--num-nodes", fmt.Sprint(defaultNodesPerZone+1), // 2 nodes/zone avoids autoscaling during TestClusterScaleUp
 		"--min-nodes", fmt.Sprint(defaultNodesPerZone),
-		"--max-nodes", fmt.Sprint(defaultNodesPerZone + 1), // Needed for scaling cluster
+		"--max-nodes", fmt.Sprint(defaultNodesPerZone + 2), // headroom above initial count
 		"--enable-autoscaling", // Enable autoscaling
 		"--autoprovisioning-network-tags", strings.Join([]string{autoprovisioningNodeTag}, ","),
 		"--machine-type", gcpDefaultMachineType,
 		"--disk-size", "30GB", // Limit disk size to 30GB
 		"--quiet", // Suppress interactive prompts
 	}
 
+	if sa := getNodeServiceAccount(); sa != "" {
+		args = append(args, "--service-account", sa)
+	}
+
 	cmd := exec.Command("gcloud", args...)
 
 	// Stream gcloud's stdout/stderr directly for real-time visibility into the long-running creation process.

tests/e2e/operator/region.go

@@ -34,6 +34,7 @@ const (
 	testOperatorRepo      = "cockroachdb-operator"
 	testInitContainerRepo = "init-container"
 	testInotifywaitRepo   = "inotifywait"
+	DefaultClusterDomain = "cluster.local"
 )
 
 var (
@@ -136,9 +137,14 @@ func (r *Region) InstallCharts(t *testing.T, cluster string, index int) {
 		InstallCockroachDBEnterpriseOperator(t, kubectlOptions, r.RegionCodes[index])
 	}
 
+	clusterDomain := DefaultClusterDomain
+	if r.IsMultiRegion {
+		clusterDomain = CustomDomains[index]
+	}
+
 	if r.IsCertManager {
 		crdbOp = PatchHelmValues(map[string]string{
-			"cockroachdb.clusterDomain":               CustomDomains[index],
+			"cockroachdb.clusterDomain":               clusterDomain,
 			"cockroachdb.tls.enabled":                 "true",
 			"cockroachdb.tls.selfSigner.enabled":      "false",
 			"cockroachdb.tls.certManager.enabled":     "true",
@@ -147,7 +153,7 @@ func (r *Region) InstallCharts(t *testing.T, cluster string, index int) {
 		})
 	} else {
 		crdbOp = PatchHelmValues(map[string]string{
-			"cockroachdb.clusterDomain":             CustomDomains[index],
+			"cockroachdb.clusterDomain":             clusterDomain,
 			"cockroachdb.tls.enabled":               "true",
 			"cockroachdb.tls.selfSigner.caProvided": "true",
 			"cockroachdb.tls.selfSigner.caSecret":   customCASecret,
@@ -165,6 +171,11 @@ func (r *Region) InstallCharts(t *testing.T, cluster string, index int) {
 
 	helm.Install(t, crdbOptions, helmChartPath, ReleaseName)
 
+	k8s.WaitUntilServiceAvailable(t, kubectlOptions, "cockroachdb-join", 30, 5*time.Second)
+	err := k8s.RunKubectlE(t, kubectlOptions, "patch", "service", "cockroachdb-join",
+		"--type=merge", "-p", `{"spec":{"publishNotReadyAddresses":true}}`)
+	require.NoError(t, err)
+
 	serviceName := "cockroachdb-public"
 	k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 30, 5*time.Second)
 }
@@ -357,6 +368,8 @@ func (r *Region) ValidateCRDBContainerResources(t *testing.T, kubectlOptions *k8
 
 // CreateCACertificate creates CA cert and key at the same path.
 func (r *Region) CreateCACertificate(t *testing.T) error {
+	r.CleanUpCACertificate(t)
+
 	// Create CA secret in all regions.
 	cmd := shell.Command{
 		Command:    "cockroach",
@@ -487,10 +500,14 @@ func (r *Region) createOperatorRegions(index int, nodes int, customDomains map[i
 			"nodes":         nodes,
 			"namespace":     r.Namespace[r.Clusters[i]],
 		}
-		if len(r.Clusters) > i && r.Clusters[i] != "" {
-			if domain, ok := customDomains[i]; ok {
-				region["domain"] = domain
+		if r.IsMultiRegion {
+			if len(r.Clusters) > i && r.Clusters[i] != "" {
+				if domain, ok := customDomains[i]; ok {
+					region["domain"] = domain
+				}
 			}
+		} else {
+			region["domain"] = DefaultClusterDomain
 		}
 		return region
 	}
@@ -786,8 +803,12 @@ func (r *Region) BaseRegionConfig(cluster string, index int) map[string]interfac
 		"nodes":         r.NodeCount,
 		"namespace":     r.Namespace[cluster],
 	}
-	if domain, ok := CustomDomains[index]; ok {
-		region["domain"] = domain
+	if r.IsMultiRegion {
+		if domain, ok := CustomDomains[index]; ok {
+			region["domain"] = domain
+		}
+	} else {
+		region["domain"] = DefaultClusterDomain
 	}
 	return region
 }
@@ -909,9 +930,14 @@ func (r *Region) InstallChartsWithAdvancedConfig(t *testing.T, cluster string, i
 		}
 	}
 
+	clusterDomain := DefaultClusterDomain
+	if r.IsMultiRegion {
+		clusterDomain = CustomDomains[index]
+	}
+
 	// Build helm values
 	helmValues := PatchHelmValues(map[string]string{
-		"cockroachdb.clusterDomain":             CustomDomains[index],
+		"cockroachdb.clusterDomain":             clusterDomain,
 		"cockroachdb.tls.selfSigner.caProvided": "true",
 		"cockroachdb.tls.selfSigner.caSecret":   customCASecret,
 	})

tests/testutil/require.go

@@ -57,8 +57,8 @@ func RequireClusterToBeReadyEventuallyTimeout(t *testing.T, crdbCluster Cockroac
 		func(ctx context.Context) (bool, error) {
 			ss, err := fetchStatefulSet(crdbCluster.K8sClient, crdbCluster.StatefulSetName, crdbCluster.Namespace)
 			if err != nil {
-				t.Logf("error fetching stateful set")
-				return false, err
+				t.Logf("transient error fetching stateful set, will retry: %v", err)
+				return false, nil
 			}
 
 			if ss == nil {
@@ -87,7 +87,8 @@ func RequireCRDBClusterToBeReadyEventuallyTimeout(t *testing.T, opts *k8s.Kubect
 				LabelSelector: "app=cockroachdb",
 			})
 			if err != nil {
-				return false, err
+				t.Logf("transient error listing pods, will retry: %v", err)
+				return false, nil
 			}
 
 			if len(pods) != crdbCluster.DesiredNodes {