Skip to content

Commit 71b82df

Browse files
committed
cleanup
1 parent 47a0b1c commit 71b82df

2 files changed

Lines changed: 9 additions & 57 deletions

File tree

tests/e2e/operator/infra/azure.go

Lines changed: 8 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,13 @@ import (
2222
"sigs.k8s.io/controller-runtime/pkg/client/config"
2323
)
2424

25-
// ─── AZURE CONSTANTS ──────────────────────────────────────────────────────────
26-
2725
const (
2826
azureDefaultNodeVMSize = "Standard_D4s_v3"
2927
azureDefaultMaxPods = 30
3028

31-
// Environment variables consumed by azure.go — mirror GCP's pattern.
3229
envAzureSubscriptionID = "AZURE_SUBSCRIPTION_ID"
3330
envAzureClientID = "AZURE_CLIENT_ID"
34-
envAzureClientSecret = "AZURE_CLIENT_SECRET" // #nosec G101 - env var name, not a credential
31+
envAzureClientSecret = "AZURE_CLIENT_SECRET"
3532
envAzureTenantID = "AZURE_TENANT_ID"
3633
envAzureResourcePrefix = "AZURE_RESOURCE_PREFIX"
3734
)
@@ -67,8 +64,6 @@ var azureClusterConfigTemplates = []AzureClusterConfig{
6764
},
6865
}
6966

70-
// ─── AZURE REGION ─────────────────────────────────────────────────────────────
71-
7267
// AzureRegion implements CloudProvider for AKS clusters on Azure.
7368
type AzureRegion struct {
7469
*operator.Region
@@ -82,8 +77,6 @@ type AzureRegion struct {
8277
kubeConfigPath string
8378
}
8479

85-
// ─── CLOUD PROVIDER INTERFACE ─────────────────────────────────────────────────
86-
8780
// SetUpInfra creates Azure infrastructure: resource group, VNets, subnets, AKS clusters,
8881
// VNet peering (for multi-region), and deploys CoreDNS.
8982
func (r *AzureRegion) SetUpInfra(t *testing.T) {
@@ -265,8 +258,6 @@ func (r *AzureRegion) CanScale() bool {
265258
return true
266259
}
267260

268-
// ─── RESOURCE CREATION ────────────────────────────────────────────────────────
269-
270261
// createResourceGroup creates an Azure resource group.
271262
func (r *AzureRegion) createResourceGroup(t *testing.T, location string) error {
272263
cmd := exec.Command("az", "group", "create",
@@ -416,8 +407,6 @@ func createAKSCluster(t *testing.T, resourceGroup, clusterName string, cfg Azure
416407
return nil
417408
}
418409

419-
// ─── VNET PEERING ─────────────────────────────────────────────────────────────
420-
421410
// setupVNetPeering creates bidirectional VNet peering between the two cluster VNets
422411
// so that pods and services (including CoreDNS LB IPs) can communicate cross-cluster.
423412
func (r *AzureRegion) setupVNetPeering(t *testing.T) error {
@@ -473,28 +462,8 @@ func (r *AzureRegion) setupVNetPeering(t *testing.T) error {
473462
return nil
474463
}
475464

476-
// ─── COREDNS (AZURE-SPECIFIC) ─────────────────────────────────────────────────
477-
//
478-
// AKS manages its own CoreDNS Deployment. We must NOT replace it (its selector
479-
// is immutable) and must NOT write to its `coredns` ConfigMap (that would break
480-
// cluster DNS). Instead we use two AKS-native mechanisms:
481-
//
482-
// 1. `coredns-custom` ConfigMap — AKS CoreDNS watches this for custom
483-
// forwarding/rewrite rules (keys ending in .server or .override).
484-
// 2. A separate `crl-core-dns` LoadBalancer Service whose selector is
485-
// dynamically detected (either `k8s-app: kube-dns` or `k8s-app: coredns`)
486-
// depending on the AKS version. Using the wrong label results in an LB
487-
// with no endpoints and silent DNS failures.
488-
//
489-
// Other clusters query the `crl-core-dns` LB IP → hits AKS CoreDNS pods →
490-
// which apply the forwarding rules from `coredns-custom` → cross-cluster DNS works.
491-
492-
// deployAndConfigureCoreDNS is the Azure-specific CoreDNS setup. It:
493-
// 1. Applies the `coredns-custom` ConfigMap with placeholder IPs.
494-
// 2. Creates the `crl-core-dns` internal LB Service targeting AKS CoreDNS pods.
495-
// 3. Restarts AKS CoreDNS so it picks up the new ConfigMap immediately.
496-
// 4. Waits for the LB to be assigned an IP, then records it.
497-
// 5. Does a final pass to update all clusters with the real cross-cluster IPs.
465+
// deployAndConfigureCoreDNS sets up cross-cluster DNS on AKS using the coredns-custom
466+
// ConfigMap and a crl-core-dns internal LoadBalancer Service targeting AKS CoreDNS pods.
498467
func (r *AzureRegion) deployAndConfigureCoreDNS(t *testing.T, kubeConfigPath string) error {
499468
for i, clusterName := range r.Clusters {
500469
kubectlOpts := k8s.NewKubectlOptions(clusterName, kubeConfigPath, coreDNSNamespace)
@@ -614,16 +583,9 @@ func buildAzureCoreDNSCustomData(thisDomain string, allClusters map[string]cored
614583
return data
615584
}
616585

617-
// detectCoreDNSPodLabel dynamically detects the label used by AKS CoreDNS pods.
618-
// AKS versions use either "k8s-app: kube-dns" (standard Kubernetes DNS label, most common)
619-
// or "k8s-app: coredns" (seen in some AKS configurations). We try both and use whichever
620-
// has running pods. Using the wrong label results in an LB with no endpoints, silently
621-
// breaking cross-cluster DNS forwarding.
622-
//
623-
// Uses kubectl (not the Go k8s client) because corporate Netskope proxies sometimes
624-
// return "Bad Gateway" for Go k8s client connections while kubectl succeeds (kubectl
625-
// uses the macOS system keychain which includes the Netskope CA, so it can verify
626-
// Netskope's intercepted certificate). Verified: AKS uses k8s-app=kube-dns.
586+
// detectCoreDNSPodLabel returns the k8s-app label used by AKS CoreDNS pods (either
587+
// "kube-dns" or "coredns" depending on AKS version). Uses kubectl instead of the Go
588+
// k8s client to avoid Netskope proxy "Bad Gateway" errors on corporate networks.
627589
func detectCoreDNSPodLabel(t *testing.T, kubectlOpts *k8s.KubectlOptions) map[string]string {
628590
for _, label := range []string{"kube-dns", "coredns"} {
629591
output, err := k8s.RunKubectlAndGetOutputE(t, kubectlOpts,
@@ -689,8 +651,6 @@ func applyAzureCoreDNSService(t *testing.T, kubectlOpts *k8s.KubectlOptions) err
689651
return nil
690652
}
691653

692-
// ─── HELPERS ──────────────────────────────────────────────────────────────────
693-
694654
// getResourcePrefix returns the prefix used for all Azure resource names (resource groups,
695655
// VNets, clusters). Set AZURE_RESOURCE_PREFIX to override the default prefix.
696656
// Defaults to "shreyaskm" to make resources easy to identify and clean up in a shared subscription.
@@ -750,17 +710,8 @@ func ensureAzureLogin(t *testing.T) error {
750710
return nil
751711
}
752712

753-
// UpdateKubeconfigAzure fetches AKS credentials and adds them to the local kubeconfig
754-
// under the given context alias (--context sets the name directly, unlike gcloud).
755-
//
756-
// After merging credentials, insecure-skip-tls-verify is set on the cluster entry.
757-
// This is required because corporate Netskope TLS inspection proxies intercept HTTPS
758-
// connections to the AKS API server and replace the server certificate with one signed
759-
// by the Netskope CA (ca.cockroachlabs.goskope.com). kubectl works because macOS
760-
// has the Netskope CA installed in the system keychain, but the Go k8s client used by
761-
// terratest only trusts the certificate-authority-data in the kubeconfig (the cluster's
762-
// self-signed CA). Enabling insecure-skip-tls-verify is the appropriate workaround for
763-
// test clusters in this corporate network environment.
713+
// UpdateKubeconfigAzure fetches AKS credentials into the kubeconfig under the given
714+
// context alias and sets insecure-skip-tls-verify to handle Netskope TLS inspection.
764715
func UpdateKubeconfigAzure(t *testing.T, resourceGroup, clusterName, alias string) error {
765716
args := []string{
766717
"aks", "get-credentials",

tests/e2e/operator/region.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,7 @@ func (r *Region) EnsureKubeConfigPath() (string, error) {
473473
// Any failure in doing so might cause issues in other tests as some of the
474474
// cluster resources are tied to the namespace.
475475
func (r *Region) CleanupResources(t *testing.T) {
476+
kubeConfig, _ := r.GetCurrentContext(t)
476477
for cluster, namespace := range r.Namespace {
477478
kubectlOptions := k8s.NewKubectlOptions(cluster, kubeConfig, namespace)
478479
certManagerK8sOptions := k8s.NewKubectlOptions(cluster, kubeConfig, testutil.CertManagerNamespace)

0 commit comments

Comments
 (0)