Use OCP registries.conf for disconnected deployments

This change adds functionality to check for ImageContentSourcePolicy in the OCP cluster.
If we find it, we infer from it's existance that we are running in a disconnected OCP
deployment. If this is indeed the case, we will pull the registries.conf file
from the MachineConfig and use this for our dataplane nodes configured via
edpm_ansible.

Signed-off-by: Brendan Shephard <bshephar@redhat.com>

Update RBAC for ICSP and MachineConfig

This change allows the OpenStackDataPlaneNodeSet controller to get and list
ImageContentSourcePolicy and MachineConfig CR's from the cluster. This facilitates
integration of the dataplane with the OCP registries.conf configuration to ensure
consistency.

Signed-off-by: Brendan Shephard <bshephar@redhat.com>

Return err if no prefix

Signed-off-by: Brendan Shephard <bshephar@redhat.com>

Author: Brendan Shephard

apis/go.mod

@@ -35,13 +35,15 @@ require (
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
 	golang.org/x/tools v0.24.0 // indirect
-	k8s.io/api v0.29.9
 	k8s.io/apimachinery v0.29.9
 	k8s.io/client-go v0.29.9
 	sigs.k8s.io/controller-runtime v0.17.6
 )
 
-require k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+require (
+	k8s.io/api v0.29.9
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+)
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect

config/rbac/role.yaml

@@ -499,6 +499,13 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - machineconfiguration.openshift.io
+  resources:
+  - machineconfig
+  verbs:
+  - get
+  - list
 - apiGroups:
   - manila.openstack.org
   resources:
@@ -647,6 +654,13 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - operator.openshift.io
+  resources:
+  - imagecontentsourcepolicy
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ovn.openstack.org
   resources:

controllers/dataplane/openstackdataplanenodeset_controller.go

@@ -120,6 +120,10 @@ func (r *OpenStackDataPlaneNodeSetReconciler) GetLogger(ctx context.Context) log
 // +kubebuilder:rbac:groups="image.openshift.io",resources=imagetags,verbs=get;list;watch
 // +kubebuilder:rbac:groups="image.openshift.io",resources=imagestreamtags,verbs=get;list;watch
 
+// RBAC for ImageContentSourcePolicy and MachineConfig
+// +kubebuilder:rbac:groups="operator.openshift.io",resources=imagecontentsourcepolicy,verbs=get;list
+// +kubebuilder:rbac:groups="machineconfiguration.openshift.io",resources=machineconfig,verbs=get;list
+
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 // TODO(user): Modify the Reconcile function to compare the state specified by
@@ -565,7 +569,8 @@ func checkDeployment(ctx context.Context, helper *helper.Helper,
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager(
-	ctx context.Context, mgr ctrl.Manager) error {
+	ctx context.Context, mgr ctrl.Manager,
+) error {
 	// index for ConfigMaps listed on ansibleVarsFrom
 	if err := mgr.GetFieldIndexer().IndexField(ctx,
 		&dataplanev1.OpenStackDataPlaneNodeSet{}, "spec.ansibleVarsFrom.ansible.configMaps",

main.go

@@ -77,6 +77,8 @@ import (
 	dataplanev1 "github.com/openstack-k8s-operators/openstack-operator/apis/dataplane/v1beta1"
 
 	ocp_configv1 "github.com/openshift/api/config/v1"
+	machineconfig "github.com/openshift/api/machineconfiguration/v1"
+	ocp_image "github.com/openshift/api/operator/v1alpha1"
 	clientcontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/client"
 	corecontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/core"
 	dataplanecontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/dataplane"
@@ -120,6 +122,8 @@ func init() {
 	utilruntime.Must(certmgrv1.AddToScheme(scheme))
 	utilruntime.Must(barbicanv1.AddToScheme(scheme))
 	utilruntime.Must(ocp_configv1.AddToScheme(scheme))
+	utilruntime.Must(ocp_image.AddToScheme(scheme))
+	utilruntime.Must(machineconfig.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
 

pkg/dataplane/inventory.go

@@ -38,8 +38,7 @@ import (
 
 // getAnsibleVarsFrom gets ansible vars from ConfigMap/Secret
 func getAnsibleVarsFrom(ctx context.Context, helper *helper.Helper, namespace string, ansible *dataplanev1.AnsibleOpts) (map[string]interface{}, error) {
-
-	var result = make(map[string]interface{})
+	result := make(map[string]interface{})
 	var value interface{}
 
 	for _, dataSource := range ansible.AnsibleVarsFrom {
@@ -90,7 +89,8 @@ func GenerateNodeSetInventory(ctx context.Context, helper *helper.Helper,
 	instance *dataplanev1.OpenStackDataPlaneNodeSet,
 	allIPSets map[string]infranetworkv1.IPSet, dnsAddresses []string,
 	containerImages openstackv1.ContainerImages,
-	netServiceNetMap map[string]string) (string, error) {
+	netServiceNetMap map[string]string,
+) (string, error) {
 	inventory := ansible.MakeInventory()
 	nodeSetGroup := inventory.AddGroup(instance.Name)
 	groupVars, err := getAnsibleVarsFrom(ctx, helper, instance.Namespace, &instance.Spec.NodeTemplate.Ansible)
@@ -111,6 +111,23 @@ func GenerateNodeSetInventory(ctx context.Context, helper *helper.Helper,
 	// add the NodeSet name variable
 	nodeSetGroup.Vars["edpm_nodeset_name"] = instance.Name
 
+	isDisconnected, err := util.IsDisconnectedOCP(ctx, helper)
+	if err != nil {
+		return "", err
+	}
+
+	if isDisconnected {
+		registryConfig, err := util.GetMCRegistryConf(ctx, helper)
+		if err != nil {
+			return "", err
+		}
+		helper.GetLogger().Info("disconnected registry was identified via the ImageContentSourcePolicy. Using OCP registry.")
+
+		// Use OCP registries.conf for disconnected deployments
+		nodeSetGroup.Vars["edpm_podman_registries_conf"] = registryConfig
+		nodeSetGroup.Vars["edpm_podman_disconnected_ocp"] = isDisconnected
+	}
+
 	// add TLS ansible variable
 	nodeSetGroup.Vars["edpm_tls_certs_enabled"] = instance.Spec.TLSEnabled
 	if instance.Spec.Tags != nil {
@@ -205,7 +222,8 @@ func GenerateNodeSetInventory(ctx context.Context, helper *helper.Helper,
 // populateInventoryFromIPAM populates inventory from IPAM
 func populateInventoryFromIPAM(
 	ipSet *infranetworkv1.IPSet, host ansible.Host,
-	dnsAddresses []string, hostName string) {
+	dnsAddresses []string, hostName string,
+) {
 	var dnsSearchDomains []string
 	for _, res := range ipSet.Status.Reservation {
 		// Build the vars for ips/routes etc
@@ -243,8 +261,8 @@ func populateInventoryFromIPAM(
 // set group ansible vars from NodeTemplate
 func resolveGroupAnsibleVars(template *dataplanev1.NodeTemplate, group *ansible.Group,
 	containerImages openstackv1.ContainerImages,
-	netServiceNetMap map[string]string) error {
-
+	netServiceNetMap map[string]string,
+) error {
 	if template.Ansible.AnsibleUser != "" {
 		group.Vars["ansible_user"] = template.Ansible.AnsibleUser
 	}
@@ -320,8 +338,8 @@ func resolveGroupAnsibleVars(template *dataplanev1.NodeTemplate, group *ansible.
 
 // set host ansible vars from NodeSection
 func resolveHostAnsibleVars(node *dataplanev1.NodeSection,
-	host *ansible.Host, netServiceNetMap map[string]string) error {
-
+	host *ansible.Host, netServiceNetMap map[string]string,
+) error {
 	if node.Ansible.AnsibleUser != "" {
 		host.Vars["ansible_user"] = node.Ansible.AnsibleUser
 	}
@@ -344,13 +362,12 @@ func resolveHostAnsibleVars(node *dataplanev1.NodeSection,
 		host.Vars["network_servicenet_map"] = serviceNetMap
 	}
 	return nil
-
 }
 
 // unmarshal raw strings into an ansible vars dictionary
 func unmarshalAnsibleVars(ansibleVars map[string]json.RawMessage,
-	parsedVars map[string]interface{}) error {
-
+	parsedVars map[string]interface{},
+) error {
 	for key, val := range ansibleVars {
 		var v interface{}
 		err := yaml.Unmarshal(val, &v)
@@ -363,7 +380,8 @@ func unmarshalAnsibleVars(ansibleVars map[string]json.RawMessage,
 }
 
 func buildNetworkVars(networks []infranetworkv1.IPSetNetwork,
-	netServiceNetMap map[string]string) ([]string, map[string]string) {
+	netServiceNetMap map[string]string,
+) ([]string, map[string]string) {
 	serviceNetMap := make(map[string]string)
 	var nets []string
 	for _, network := range networks {

pkg/dataplane/util/image_registry.go

@@ -0,0 +1,131 @@
+package util
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	mc "github.com/openshift/api/machineconfiguration/v1"
+	ocpimage "github.com/openshift/api/operator/v1alpha1"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// machineConfigIgnition - holds the relevant parts of the ignition file that we need to create
+// the registries.conf on the dataplane nodes.
+type machineConfigIgnition struct {
+	Ignition struct {
+		Version string `json:"version"`
+	} `json:"ignition"`
+	Storage struct {
+		Files []struct {
+			Contents struct {
+				Compression string `json:"compression,omitempty"`
+				Source      string `json:"source"`
+			} `json:"contents"`
+			Mode      int    `json:"mode,omitempty"`
+			Overwrite bool   `json:"overwrite,omitempty"`
+			Path      string `json:"path"`
+		} `json:"files"`
+	} `json:"storage"`
+}
+
+// IsDisconnectedOCP - Will retrieve a ImageContentSourcePolicyList. If the list is not
+// empty, we can infer that the OCP cluster is a disconnected deployment.
+func IsDisconnectedOCP(ctx context.Context, helper *helper.Helper) (bool, error) {
+	icspList := ocpimage.ImageContentSourcePolicyList{}
+
+	listOpts := []client.ListOption{}
+	err := helper.GetClient().List(ctx, &icspList, listOpts...)
+	if err != nil {
+		return false, err
+	}
+
+	if len(icspList.Items) != 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// GetMCRegistryConf - will unmarshal the MachineConfig ignition file the machineConfigIgnition object.
+// This is then parsed and the base64 decoded string is returned.
+func GetMCRegistryConf(ctx context.Context, helper *helper.Helper) (string, error) {
+	var registriesConf string
+
+	masterMachineConfig, err := getMachineConfig(ctx, helper)
+	if err != nil {
+		return registriesConf, err
+	}
+
+	config := machineConfigIgnition{}
+	registriesConf, err = config.formatRegistriesConfString(&masterMachineConfig)
+	if err != nil {
+		return registriesConf, err
+	}
+
+	return registriesConf, nil
+}
+
+func (mci *machineConfigIgnition) removePrefixFromB64String() (string, error) {
+	const b64Prefix string = "data:text/plain;charset=utf-8;base64,"
+	if strings.HasPrefix(mci.Storage.Files[0].Contents.Source, b64Prefix) {
+		return mci.Storage.Files[0].Contents.Source[len(b64Prefix):], nil
+	}
+
+	return "", fmt.Errorf("no b64prefix found in MachineConfig")
+}
+
+func (mci *machineConfigIgnition) formatRegistriesConfString(machineConfig *mc.MachineConfig) (string, error) {
+	var (
+		err             error
+		rawConfigString string
+		configString    []byte
+	)
+
+	err = json.Unmarshal([]byte(machineConfig.Spec.Config.Raw), &mci)
+	if err != nil {
+		return "", err
+	}
+
+	rawConfigString, err = mci.removePrefixFromB64String()
+	if err != nil {
+		return "", err
+	}
+	configString, err = base64.StdEncoding.DecodeString(rawConfigString)
+	if err != nil {
+		return "", err
+	}
+
+	return string(configString), nil
+}
+
+func masterMachineConfigBuilder(machineConfigRegistries string) mc.MachineConfig {
+	return mc.MachineConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      machineConfigRegistries,
+			Namespace: "",
+		},
+	}
+}
+
+func getMachineConfig(ctx context.Context, helper *helper.Helper) (mc.MachineConfig, error) {
+	const machineConfigRegistries string = "99-master-generated-registries"
+
+	masterMachineConfig := masterMachineConfigBuilder(machineConfigRegistries)
+
+	err := helper.GetClient().Get(ctx,
+		types.NamespacedName{
+			Name: masterMachineConfig.Name, Namespace: masterMachineConfig.Namespace,
+		}, &masterMachineConfig)
+	if err != nil {
+		return masterMachineConfig, err
+	}
+
+	return masterMachineConfig, nil
+}