fix: use one big vnet and attach AKS clusters to it to avoid creating bastion multiple times by awesomenix · Pull Request #8646 · Azure/AgentBaker

awesomenix · 2026-06-05T18:40:59Z

Make everyone's life simple
README has all the details, but idea is to use shared VNET and stop doing unnecessary activities multiple times.

Copilot

Pull request overview

This PR refactors the E2E harness to use a single shared per-location VNet and Bastion (in abe2e-{location}) and attaches all AKS clusters to per-cluster subnets inside that VNet, reducing repeated Bastion provisioning and centralizing shared infrastructure creation.

Changes:

Add shared-infrastructure provisioning (shared_infra.go) to create/ensure a shared VNet, Bastion, Firewall, and managed identity, plus auto-allocation of per-cluster subnets.
Update cluster creation and networking helpers to rely on VnetSubnetID (BYO VNet/subnet) rather than discovering VNets in the MC_ resource group.
Update SSH-over-Bastion flow and documentation to reflect shared Bastion/VNet usage.

Reviewed changes

Copilot reviewed 8 out of 8 changed files in this pull request and generated 6 comments.

Show a summary per file

File	Description
e2e/vmss.go	Switch Bastion SSH command to use shared Bastion name + shared RG.
e2e/shared_infra.go	Introduce shared VNet/Bastion/Firewall/identity creation and per-cluster subnet allocation/cleanup.
e2e/README.md	Document the new shared-infra architecture and updated scenario authoring pointers.
e2e/node_config.go	Adjust tenant ID derivation to handle user-assigned identity clusters.
e2e/kube.go	Read cluster subnet ID from agent pool `VnetSubnetID` instead of listing VNets.
e2e/cluster.go	Create per-cluster subnet after name hashing; use shared Bastion; derive VNet from subnet ID.
e2e/cache.go	Ensure all cached cluster creators configure the shared VNet before preparing clusters.
e2e/aks_model.go	Avoid hardcoded subnet CIDRs; use parsed VNet/subnet info when configuring firewall/isolated settings.

+	subnetAddressPrefix := vnet.addressPrefix
+
 	subnetParameters := armnetwork.Subnet{
 		ID: to.Ptr(subnetId),
 		Properties: &armnetwork.SubnetPropertiesFormat{


Copilot

Pull request overview

Copilot reviewed 8 out of 8 changed files in this pull request and generated 6 comments.

Copilot

Pull request overview

Copilot reviewed 8 out of 8 changed files in this pull request and generated 7 comments.

Copilot

Pull request overview

Copilot reviewed 8 out of 8 changed files in this pull request and generated 7 comments.

Copilot

Pull request overview

Copilot reviewed 10 out of 10 changed files in this pull request and generated 9 comments.

+	poller, err := config.Azure.Subnet.BeginCreateOrUpdate(ctx, rg, SharedVNetName, PESubnetName, armnetwork.Subnet{
+		Properties: &armnetwork.SubnetPropertiesFormat{
+			AddressPrefix: to.Ptr(PESubnetCIDR),
+		},
+	}, nil)


+		for _, subnet := range page.Value {
+			name := *subnet.Name
+			if !strings.HasPrefix(name, "aks-subnet-") {


+		for _, s := range page.Value {
+			if s.Properties != nil && s.Properties.AddressPrefix != nil {
+				usedCIDRs[*s.Properties.AddressPrefix] = true
+			}


+// ensureClusterIdentity creates a user-assigned managed identity for AKS clusters
+// and grants it Network Contributor on the subscription so it can manage route tables
+// in both the shared VNet and the MC_ resource groups.


+	toolkit.Logf(ctx, "using shared bastion %s in %s", SharedBastionName, sharedRG)
+	return NewBastion(config.Azure.Credential, config.Config.SubscriptionID, sharedRG, *sharedBastion.Properties.DNSName), nil


+			if zone.Name == nil || *zone.Name != privateZoneName {
+				continue
+			}
+			zoneRG := resourceGroupFromID(*zone.ID)
+			if strings.EqualFold(zoneRG, sharedRG) {
+				continue
+			}


+			toolkit.Logf(ctx, "deleting conflicting DNS zone link %s in %s/%s (points to shared VNet)", *link.Name, zoneRG, zoneName)
+			poller, err := config.Azure.VirutalNetworkLinksClient.BeginDelete(ctx, zoneRG, zoneName, *link.Name, nil)


+	if privateEndpoint.Properties == nil || len(privateEndpoint.Properties.NetworkInterfaces) == 0 {
+		return fmt.Errorf("private endpoint has no network interfaces")
+	}

-		aRecords := make([]*armprivatedns.ARecord, len(ipAddresses))
-		for i, ip := range ipAddresses {
-			aRecords[i] = &armprivatedns.ARecord{IPv4Address: &ip}
-		}
-		ttl := int64(10)
-		aRecordSet := armprivatedns.RecordSet{
-			Properties: &armprivatedns.RecordSetProperties{
-				TTL:      &ttl,
-				ARecords: aRecords,
-			},
-		}
-		_, err := config.Azure.RecordSetClient.CreateOrUpdate(ctx, nodeResourceGroup, privateZoneName, armprivatedns.RecordTypeA, *dnsConfig.Fqdn, aRecordSet, nil)
-		if err != nil {
-			return fmt.Errorf("failed to create record set: %w", err)
-		}
+	nicID := *privateEndpoint.Properties.NetworkInterfaces[0].ID
+	nicName := nicID[strings.LastIndex(nicID, "/")+1:]


+	// Each NIC IP config has a private IP and an associated FQDN from the
+	// PE's privateLinkServiceConnections. Create one A record per FQDN.
+	for _, ipConfig := range nic.Properties.IPConfigurations {


Copilot

Pull request overview

Copilot reviewed 10 out of 10 changed files in this pull request and generated 7 comments.

+	networkProfile.ServiceCidr = to.Ptr("172.16.0.0/16")
 	networkProfile.ServiceCidrs = []*string{
 		networkProfile.ServiceCidr,
 		to.Ptr("fd12:3456:789a:1::/108"),


+	poller, err := config.Azure.Subnet.BeginCreateOrUpdate(ctx, rg, SharedVNetName, PESubnetName, armnetwork.Subnet{
+		Properties: &armnetwork.SubnetPropertiesFormat{
+			AddressPrefix: to.Ptr(PESubnetCIDR),
+		},


+	existing, err := config.Azure.UserAssignedIdentities.Get(ctx, rg, SharedClusterIdentity, nil)
+	if err == nil {
+		return *existing.ID, *existing.Properties.TenantID, nil
+	}


+	toolkit.Logf(ctx, "using shared bastion %s in %s", SharedBastionName, sharedRG)
+	return NewBastion(config.Azure.Credential, config.Config.SubscriptionID, sharedRG, *sharedBastion.Properties.DNSName), nil


+	// Check if the record already exists with the correct IPs
+	existing, err := config.Azure.RecordSetClient.Get(ctx, resourceGroup, zoneName, armprivatedns.RecordTypeA, recordName, nil)
+	if err == nil && existing.Properties != nil && existing.Properties.ARecords != nil {


+	for _, ipConfig := range nic.Properties.IPConfigurations {
+		if ipConfig.Properties == nil || ipConfig.Properties.PrivateIPAddress == nil {
+			continue
+		}
+		ip := *ipConfig.Properties.PrivateIPAddress

-func addDNSZoneGroup(ctx context.Context, privateZone *armprivatedns.PrivateZone, nodeResourceGroup, privateZoneName, endpointName string) error {
-	groupName := strings.Replace(privateZoneName, ".", "-", -1) // replace . with -
-	_, err := config.Azure.PrivateDNSZoneGroup.Get(ctx, nodeResourceGroup, endpointName, groupName, nil)
-	if err == nil {
-		return nil
-	}
-	dnsZonegroup := armnetwork.PrivateDNSZoneGroup{
-		Name: to.Ptr(fmt.Sprintf("%s/default", privateZoneName)),
-		Properties: &armnetwork.PrivateDNSZoneGroupPropertiesFormat{
-			PrivateDNSZoneConfigs: []*armnetwork.PrivateDNSZoneConfig{{
-				Name: to.Ptr(groupName),
-				Properties: &armnetwork.PrivateDNSZonePropertiesFormat{
-					PrivateDNSZoneID: privateZone.ID,
+		// The PE's CustomDNSConfigs or PrivateLinkServiceConnections tell us the
+		// FQDN, but they may be empty. Use the IP config's
+		// PrivateLinkConnectionProperties for the FQDN list.
+		if ipConfig.Properties.PrivateLinkConnectionProperties == nil || len(ipConfig.Properties.PrivateLinkConnectionProperties.Fqdns) == 0 {
+			continue


      - go.mod
      - go.sum
-      - e2e/
+      - e2e/scenario_win_test.go


… bastion multiple times

Devinwong

e

Devinwong · 2026-06-08T19:46:58Z

+		return fmt.Errorf("checking subnet %s: %w", subnetName, err)
+	}
+
+	return retryOn409(ctx, fmt.Sprintf("creating subnet %s", subnetName), func() error {


Condsider adding a retry on 400 too?
Azure returns HTTP 400 (Bad Request) for overlapping subnet address spaces, not 409 Conflict.

if two tests try to create the same subnet name(deterministic from cluster name), only one wins. But if two different clusters hash to the same starting slot and list subnets at the same time, they could both pick the same free CIDR and one will get aconflict on a different subnet name with the same address space. Azure will reject this with a "subnet address space overlaps" error, which is not a 400

its impossible to have 2 different clusters to have same hash, because the hash is determined by the public SSH key. WE should be ok here

Devinwong · 2026-06-08T19:48:01Z

+	}
+
+	// Best-effort cleanup of orphaned cluster subnets
+	cleanupOrphanedSubnets(ctx, rg)


async for unblocking every test?

deferring to next PR

Copilot AI review requested due to automatic review settings June 5, 2026 18:40

awesomenix requested review from AbelHu, Devinwong, SriHarsha001, calvin197, cameronmeissner, djsly, ganeshkumarashok, lilypan26, mxj220, pdamianov-dev, phealy, r2k1, sulixu, surajssd, timmy-wright and zachary-bailey as code owners June 5, 2026 18:41

Copilot started reviewing on behalf of awesomenix June 5, 2026 18:41 View session

Copilot AI reviewed Jun 5, 2026

View reviewed changes

Copilot AI review requested due to automatic review settings June 5, 2026 20:54

awesomenix force-pushed the nishp/fast/prefetchopt branch from 2579b61 to 738f4ae Compare June 5, 2026 20:54

Copilot started reviewing on behalf of awesomenix June 5, 2026 20:54 View session

Copilot AI reviewed Jun 5, 2026

View reviewed changes

Comment thread e2e/aks_model.go Outdated

Comment thread e2e/shared_infra.go

Comment thread e2e/shared_infra.go

Comment thread e2e/node_config.go Outdated

Comment thread e2e/shared_infra.go

Comment thread e2e/README.md

awesomenix force-pushed the nishp/fast/prefetchopt branch from 738f4ae to 69c05f8 Compare June 5, 2026 21:00

Copilot AI review requested due to automatic review settings June 5, 2026 23:02

awesomenix force-pushed the nishp/fast/prefetchopt branch from 69c05f8 to f928610 Compare June 5, 2026 23:02

Copilot started reviewing on behalf of awesomenix June 5, 2026 23:02 View session

Copilot AI reviewed Jun 5, 2026

View reviewed changes

Comment thread e2e/shared_infra.go

Comment thread e2e/shared_infra.go

Comment thread e2e/cluster.go

Comment thread e2e/node_config.go Outdated

Comment thread e2e/aks_model.go

Comment thread e2e/aks_model.go

Comment thread e2e/README.md Outdated

awesomenix force-pushed the nishp/fast/prefetchopt branch from f928610 to be31cd4 Compare June 6, 2026 00:55

Copilot AI review requested due to automatic review settings June 6, 2026 13:36

awesomenix force-pushed the nishp/fast/prefetchopt branch from be31cd4 to e71ee5d Compare June 6, 2026 13:36

Copilot started reviewing on behalf of awesomenix June 6, 2026 13:36 View session

Copilot AI reviewed Jun 6, 2026

View reviewed changes

Comment thread e2e/shared_infra.go

Comment thread e2e/shared_infra.go

Comment thread e2e/aks_model.go Outdated

Comment thread e2e/node_config.go Outdated

Comment thread e2e/README.md

Comment thread e2e/README.md

Comment thread e2e/aks_model.go

awesomenix force-pushed the nishp/fast/prefetchopt branch from e71ee5d to 8315a13 Compare June 6, 2026 15:07

Copilot AI review requested due to automatic review settings June 6, 2026 21:54

awesomenix force-pushed the nishp/fast/prefetchopt branch from 8315a13 to 1bb4eab Compare June 6, 2026 21:54

Copilot started reviewing on behalf of awesomenix June 6, 2026 21:54 View session

Copilot AI reviewed Jun 6, 2026

View reviewed changes

awesomenix force-pushed the nishp/fast/prefetchopt branch from 1bb4eab to 0648508 Compare June 7, 2026 04:04

Copilot AI review requested due to automatic review settings June 7, 2026 04:21

awesomenix force-pushed the nishp/fast/prefetchopt branch from 0648508 to 7f6b2a6 Compare June 7, 2026 04:21

Copilot started reviewing on behalf of awesomenix June 7, 2026 04:21 View session

Copilot AI reviewed Jun 7, 2026

View reviewed changes

fix: use one big vnet and attach AKS clusters to it to avoid creating…

9b2d401

… bastion multiple times

awesomenix force-pushed the nishp/fast/prefetchopt branch from 7f6b2a6 to 9b2d401 Compare June 7, 2026 15:07

Devinwong reviewed Jun 8, 2026

View reviewed changes

Comment thread e2e/node_config.go

Devinwong approved these changes Jun 8, 2026

View reviewed changes

awesomenix merged commit 7621ec2 into main Jun 8, 2026
30 checks passed

awesomenix deleted the nishp/fast/prefetchopt branch June 8, 2026 23:39

		toolkit.Logf(ctx, "using shared bastion %s in %s", SharedBastionName, sharedRG)
		return NewBastion(config.Azure.Credential, config.Config.SubscriptionID, sharedRG, *sharedBastion.Properties.DNSName), nil

		toolkit.Logf(ctx, "deleting conflicting DNS zone link %s in %s/%s (points to shared VNet)", *link.Name, zoneRG, zoneName)
		poller, err := config.Azure.VirutalNetworkLinksClient.BeginDelete(ctx, zoneRG, zoneName, *link.Name, nil)

Uh oh!

Conversation

awesomenix commented Jun 5, 2026

Uh oh!

Copilot AI left a comment

Choose a reason for hiding this comment

Pull request overview

Reviewed changes

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Copilot AI left a comment

Choose a reason for hiding this comment

Pull request overview

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Copilot AI left a comment

Choose a reason for hiding this comment

Pull request overview

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Copilot AI left a comment

Choose a reason for hiding this comment

Pull request overview

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Copilot AI left a comment

Choose a reason for hiding this comment

Pull request overview

Uh oh!

Copilot AI left a comment

Choose a reason for hiding this comment

Pull request overview

Uh oh!

Devinwong left a comment • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Devinwong Jun 8, 2026

Choose a reason for hiding this comment

Uh oh!

awesomenix Jun 8, 2026

Choose a reason for hiding this comment

Uh oh!

Devinwong Jun 8, 2026

Choose a reason for hiding this comment

Uh oh!

awesomenix Jun 8, 2026

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

Devinwong left a comment •

edited

Loading