CustomDNS feature enablement along with inclusion of 4.21 upstream installer

hack/update-go-module-dependencies.sh

@@ -3,7 +3,7 @@
 export GOPRIVATE=github.com
 export GONOPROXY="y"
 
-RELEASE=release-4.20
+RELEASE=release-4.21
 VM_SKU=aro_4$(echo $RELEASE | sed 's/.*\.//')
 declare -a pinned=(
 	"github.com/openshift/assisted-service/api"

pkg/api/featureflags.go

@@ -20,4 +20,12 @@ const (
 	// Unit (MTU) on Azure virtual networks, which as of late 2021 is 3900 bytes.
 	// Otherwise cluster nodes will use the DHCP-provided MTU of 1500 bytes.
 	FeatureFlagMTU3900 = "Microsoft.RedHatOpenShift/MTU3900"
+
+	// OperatorFlagDNSType is the operator flag that controls which DNS solution to use.
+	// When set to "dnsmasq" (or absent), dnsmasq is used at the node level.
+	// When set to "clusterhosted", CoreDNS static pods are deployed via the upstream
+	// Custom DNS feature (4.21+).
+	OperatorFlagDNSType              = "aro.dns.type"
+	OperatorFlagDNSTypeDnsmasq       = "dnsmasq"
+	OperatorFlagDNSTypeClusterHosted = "clusterhosted"
 )

pkg/installer/custominstallconfig.go

@@ -14,6 +14,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"text/template"
 
 	"github.com/coreos/ignition/v2/config/util"
@@ -38,6 +39,7 @@ import (
 	"github.com/openshift/installer-aro-wrapper/pkg/cluster/graph"
 	bootstrapfiles "github.com/openshift/installer-aro-wrapper/pkg/data/bootstrap"
 	"github.com/openshift/installer-aro-wrapper/pkg/data/manifests"
+	"github.com/openshift/installer-aro-wrapper/pkg/installer/dns"
 	"github.com/openshift/installer-aro-wrapper/pkg/installer/dnsmasq"
 	"github.com/openshift/installer-aro-wrapper/pkg/installer/etchost"
 	"github.com/openshift/installer-aro-wrapper/pkg/installer/mdsd"
@@ -92,7 +94,7 @@ func (m *manager) applyInstallConfigCustomisations(ctx context.Context, installC
 		HTTPSecret:    hex.EncodeToString(httpSecret),
 	}
 
-	localdnsConfig := dnsmasq.DNSConfig{
+	localdnsConfig := dns.DNSConfig{
 		APIIntIP:  m.oc.Properties.APIServerProfile.IntIP,
 		IngressIP: m.oc.Properties.IngressProfiles[0].IP,
 	}
@@ -151,9 +153,11 @@ func (m *manager) applyInstallConfigCustomisations(ctx context.Context, installC
 	}
 
 	bootstrapAsset := g.Get(&bootstrap.Bootstrap{}).(*bootstrap.Bootstrap)
-	err = dnsmasq.CreatednsmasqIgnitionFiles(bootstrapAsset, installConfig, localdnsConfig)
-	if err != nil {
-		return nil, err
+	if m.oc.Properties.OperatorFlags[api.OperatorFlagDNSType] != api.OperatorFlagDNSTypeClusterHosted {
+		err = dnsmasq.CreatednsmasqIgnitionFiles(bootstrapAsset, installConfig, localdnsConfig)
+		if err != nil {
+			return nil, err
+		}
 	}
 	err = mdsd.AppendMdsdFiles(bootstrapAsset, bootstrapLoggingConfig)
 	if err != nil {
@@ -180,6 +184,12 @@ func (m *manager) applyInstallConfigCustomisations(ctx context.Context, installC
 	if err != nil {
 		return nil, err
 	}
+	// Inject LB IPs into Infrastructure CR for Custom DNS (CoreDNS) mode
+	if m.oc.Properties.OperatorFlags[api.OperatorFlagDNSType] == api.OperatorFlagDNSTypeClusterHosted {
+		if err = addLBIPsToInfrastructureCR(bootstrapAsset, localdnsConfig.APIIntIP, m.oc.Properties.APIServerProfile.IP, localdnsConfig.IngressIP); err != nil {
+			return nil, err
+		}
+	}
 	// Update Master and Worker Pointer Ignition with ARO API-Int IP
 	if err = replacePointerIgnition(bootstrapAsset, g, &localdnsConfig); err != nil {
 		return nil, err
@@ -313,9 +323,55 @@ func removeDNSConfigData(bootstrap *bootstrap.Bootstrap, installConfig installco
 	return nil
 }
 
+const infrastructureFilepath = "/opt/openshift/manifests/cluster-infrastructure-02-config.yml"
+
+// addLBIPsToInfrastructureCR injects API, API-Int, and Ingress load balancer IPs
+// into the Infrastructure CR within bootstrap ignition. This is needed for Custom
+// DNS (CoreDNS) mode where the MCO reads these IPs from the Infrastructure CR to
+// configure CoreDNS static pods. Mirrors upstream's addLoadBalancersToInfra()
+// in pkg/infrastructure/clusterapi/ignition.go.
+func addLBIPsToInfrastructureCR(bootstrapAsset *bootstrap.Bootstrap, apiIntIP string, apiIP string, ingressIP string) error {
+	for i, fileData := range bootstrapAsset.Config.Storage.Files {
+		if fileData.Path == infrastructureFilepath {
+			contents := strings.Split(*bootstrapAsset.Config.Storage.Files[i].Contents.Source, ",")
+			rawDecodedText, err := base64.StdEncoding.DecodeString(contents[1])
+			if err != nil {
+				return fmt.Errorf("failed to decode infrastructure CR: %w", err)
+			}
+
+			infra := &configv1.Infrastructure{}
+			if err := yaml.Unmarshal(rawDecodedText, infra); err != nil {
+				return fmt.Errorf("failed to unmarshal infrastructure CR: %w", err)
+			}
+
+			cloudLBInfo := configv1.CloudLoadBalancerIPs{
+				APIIntLoadBalancerIPs: []configv1.IP{configv1.IP(apiIntIP)},
+			}
+			if apiIP != "" {
+				cloudLBInfo.APILoadBalancerIPs = []configv1.IP{configv1.IP(apiIP)}
+			}
+			if ingressIP != "" {
+				cloudLBInfo.IngressLoadBalancerIPs = []configv1.IP{configv1.IP(ingressIP)}
+			}
+
+			infra.Status.PlatformStatus.Azure.CloudLoadBalancerConfig.ClusterHosted = &cloudLBInfo
+
+			infraContents, err := yaml.Marshal(infra)
+			if err != nil {
+				return fmt.Errorf("failed to marshal infrastructure CR: %w", err)
+			}
+
+			encoded := fmt.Sprintf("data:text/plain;charset=utf-8;base64,%s", base64.StdEncoding.EncodeToString(infraContents))
+			bootstrapAsset.Config.Storage.Files[i].Contents.Source = &encoded
+			break
+		}
+	}
+	return nil
+}
+
 // replacePointerIgnition performs the same functionality as the upstream
 // installer's pointerIgnitionConfig() but with ARO specific DNS config
-func replacePointerIgnition(a *bootstrap.Bootstrap, g graph.Graph, localdnsConfig *dnsmasq.DNSConfig) (err error) {
+func replacePointerIgnition(a *bootstrap.Bootstrap, g graph.Graph, localdnsConfig *dns.DNSConfig) (err error) {
 	masterPointerIgn := g.Get(&machine.Master{}).(*machine.Master)
 	workerPointerIgn := g.Get(&machine.Worker{}).(*machine.Worker)
 	ignitionHost := net.JoinHostPort(localdnsConfig.APIIntIP, "22623")

pkg/installer/custominstallconfig_test.go

@@ -37,6 +37,7 @@ import (
 	"github.com/openshift/installer/pkg/ipnet"
 	"github.com/openshift/installer/pkg/types"
 	azuretypes "github.com/openshift/installer/pkg/types/azure"
+	capzazure "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 
 	"github.com/openshift/installer-aro-wrapper/pkg/api"
 	"github.com/openshift/installer-aro-wrapper/pkg/bootstraplogging"
@@ -215,8 +216,18 @@ func makeInstallConfig() *installconfig.InstallConfig {
 						Region:                   "centralus",
 						NetworkResourceGroupName: "test-nrg",
 						VirtualNetwork:           "test-net",
-						ControlPlaneSubnet:       "test-cp-subnet",
-						ComputeSubnet:            "test-worker-subnet",
+						DeprecatedControlPlaneSubnet: "test-cp-subnet",
+						DeprecatedComputeSubnet:      "test-worker-subnet",
+						Subnets: []azuretypes.SubnetSpec{
+							{
+								Name: "test-cp-subnet",
+								Role: capzazure.SubnetControlPlane,
+							},
+							{
+								Name: "test-worker-subnet",
+								Role: capzazure.SubnetNode,
+							},
+						},
 						CloudName:                "AzurePublicCloud",
 						OutboundType:             azuretypes.LoadbalancerOutboundType,
 						ResourceGroupName:        "test-resource-group",
@@ -512,3 +523,127 @@ func verifyUpdateMCSCertKey(t *testing.T, bootstrap *bootstrap.Bootstrap) {
 		}
 	}
 }
+
+func makeInfrastructureCR(dnsType configv1.DNSType) []byte {
+	infra := &configv1.Infrastructure{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: configv1.SchemeGroupVersion.String(),
+			Kind:       "Infrastructure",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "cluster",
+		},
+		Status: configv1.InfrastructureStatus{
+			PlatformStatus: &configv1.PlatformStatus{
+				Azure: &configv1.AzurePlatformStatus{
+					CloudLoadBalancerConfig: &configv1.CloudLoadBalancerConfig{
+						DNSType: dnsType,
+					},
+				},
+			},
+		},
+	}
+	data, _ := yaml.Marshal(infra)
+	return data
+}
+
+func makeBootstrapWithInfraCR(infraData []byte) *bootstrap.Bootstrap {
+	encoded := "data:text/plain;charset=utf-8;base64," + base64.StdEncoding.EncodeToString(infraData)
+	b := &bootstrap.Bootstrap{}
+	b.Config = &igntypes.Config{
+		Storage: igntypes.Storage{
+			Files: []igntypes.File{
+				{
+					Node: igntypes.Node{Path: infrastructureFilepath},
+					FileEmbedded1: igntypes.FileEmbedded1{
+						Contents: igntypes.Resource{
+							Source: &encoded,
+						},
+					},
+				},
+			},
+		},
+	}
+	return b
+}
+
+func TestAddLBIPsToInfrastructureCR(t *testing.T) {
+	tests := []struct {
+		name             string
+		apiIntIP         string
+		apiIP            string
+		ingressIP        string
+		wantAPIIntIPs    []configv1.IP
+		wantAPIIPs       []configv1.IP
+		wantIngressIPs   []configv1.IP
+	}{
+		{
+			name:           "public cluster with all IPs",
+			apiIntIP:       "10.0.0.1",
+			apiIP:          "20.0.0.1",
+			ingressIP:      "10.0.0.5",
+			wantAPIIntIPs:  []configv1.IP{"10.0.0.1"},
+			wantAPIIPs:     []configv1.IP{"20.0.0.1"},
+			wantIngressIPs: []configv1.IP{"10.0.0.5"},
+		},
+		{
+			name:           "private cluster with only internal IP",
+			apiIntIP:       "10.0.0.1",
+			apiIP:          "",
+			ingressIP:      "10.0.0.5",
+			wantAPIIntIPs:  []configv1.IP{"10.0.0.1"},
+			wantAPIIPs:     nil,
+			wantIngressIPs: []configv1.IP{"10.0.0.5"},
+		},
+		{
+			name:           "cluster without ingress IP",
+			apiIntIP:       "10.0.0.1",
+			apiIP:          "20.0.0.1",
+			ingressIP:      "",
+			wantAPIIntIPs:  []configv1.IP{"10.0.0.1"},
+			wantAPIIPs:     []configv1.IP{"20.0.0.1"},
+			wantIngressIPs: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			infraData := makeInfrastructureCR(configv1.ClusterHostedDNSType)
+			bootstrapAsset := makeBootstrapWithInfraCR(infraData)
+
+			err := addLBIPsToInfrastructureCR(bootstrapAsset, tt.apiIntIP, tt.apiIP, tt.ingressIP)
+			require.NoError(t, err)
+
+			// Decode the modified Infrastructure CR from the bootstrap asset
+			contents := strings.Split(*bootstrapAsset.Config.Storage.Files[0].Contents.Source, ",")
+			decoded, err := base64.StdEncoding.DecodeString(contents[1])
+			require.NoError(t, err)
+
+			infra := &configv1.Infrastructure{}
+			err = yaml.Unmarshal(decoded, infra)
+			require.NoError(t, err)
+
+			require.NotNil(t, infra.Status.PlatformStatus.Azure.CloudLoadBalancerConfig.ClusterHosted,
+				"ClusterHosted should be set")
+			assert.Equal(t, tt.wantAPIIntIPs,
+				infra.Status.PlatformStatus.Azure.CloudLoadBalancerConfig.ClusterHosted.APIIntLoadBalancerIPs)
+			assert.Equal(t, tt.wantAPIIPs,
+				infra.Status.PlatformStatus.Azure.CloudLoadBalancerConfig.ClusterHosted.APILoadBalancerIPs)
+			assert.Equal(t, tt.wantIngressIPs,
+				infra.Status.PlatformStatus.Azure.CloudLoadBalancerConfig.ClusterHosted.IngressLoadBalancerIPs)
+		})
+	}
+}
+
+func TestAddLBIPsToInfrastructureCR_NoInfraFile(t *testing.T) {
+	// Bootstrap with no infrastructure file - function should be a no-op
+	bootstrapAsset := &bootstrap.Bootstrap{}
+	bootstrapAsset.Config = &igntypes.Config{
+		Storage: igntypes.Storage{
+			Files: []igntypes.File{},
+		},
+	}
+
+	err := addLBIPsToInfrastructureCR(bootstrapAsset, "10.0.0.1", "20.0.0.1", "10.0.0.5")
+	require.NoError(t, err, "should not error when infrastructure file is not found")
+}

pkg/installer/dns/config.go

@@ -0,0 +1,13 @@
+package dns
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+// DNSConfig holds the DNS-related network configuration used by both
+// dnsmasq and CustomDNS (CoreDNS) modes in the ARO installer wrapper.
+type DNSConfig struct {
+	APIIntIP                 string
+	IngressIP                string
+	GatewayDomains           []string
+	GatewayPrivateEndpointIP string
+}

pkg/installer/dnsmasq/dnsmasq_config.go

@@ -8,16 +8,10 @@ import (
 	"github.com/openshift/installer/pkg/asset/installconfig"
 
 	bootstrapfiles "github.com/openshift/installer-aro-wrapper/pkg/data/bootstrap"
+	"github.com/openshift/installer-aro-wrapper/pkg/installer/dns"
 )
 
-type DNSConfig struct {
-	APIIntIP                 string
-	IngressIP                string
-	GatewayDomains           []string
-	GatewayPrivateEndpointIP string
-}
-
-func CreatednsmasqIgnitionFiles(bootstrapAsset *bootstrap.Bootstrap, installConfig *installconfig.InstallConfig, dnsConfig DNSConfig) error {
+func CreatednsmasqIgnitionFiles(bootstrapAsset *bootstrap.Bootstrap, installConfig *installconfig.InstallConfig, dnsConfig dns.DNSConfig) error {
 	dnsmasqIgnConfig, err := Ignition3Config(installConfig.Config.ClusterDomain(), dnsConfig.APIIntIP, dnsConfig.IngressIP, dnsConfig.GatewayDomains, dnsConfig.GatewayPrivateEndpointIP)
 	if err != nil {
 		return err

pkg/installer/etchost/etchost_assets.go

@@ -8,10 +8,10 @@ import (
 	"github.com/openshift/installer/pkg/asset/installconfig"
 
 	bootstrapfiles "github.com/openshift/installer-aro-wrapper/pkg/data/bootstrap"
-	"github.com/openshift/installer-aro-wrapper/pkg/installer/dnsmasq"
+	"github.com/openshift/installer-aro-wrapper/pkg/installer/dns"
 )
 
-func AppendEtcHostFiles(bootstrapAsset *bootstrap.Bootstrap, installConfig installconfig.InstallConfig, dnsConfig dnsmasq.DNSConfig) error {
+func AppendEtcHostFiles(bootstrapAsset *bootstrap.Bootstrap, installConfig installconfig.InstallConfig, dnsConfig dns.DNSConfig) error {
 	etcHostIgnConfig, err := EtcHostsIgnitionConfig(installConfig.Config.ClusterDomain(), dnsConfig.APIIntIP, dnsConfig.GatewayDomains, dnsConfig.GatewayPrivateEndpointIP)
 	if err != nil {
 		return err