Merge pull request #1098 from bshephar/prototype-icsp

Use OCP registries.conf for disconnected deployments

Author: openshift-merge-bot[bot]

apis/go.mod

@@ -38,11 +38,10 @@ require (
 	k8s.io/api v0.29.9
 	k8s.io/apimachinery v0.29.9
 	k8s.io/client-go v0.29.9
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 	sigs.k8s.io/controller-runtime v0.17.6
 )
 
-require k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
-
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect

config/rbac/role.yaml

@@ -499,6 +499,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - machineconfiguration.openshift.io
+  resources:
+  - machineconfigs
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - manila.openstack.org
   resources:
@@ -647,6 +655,13 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - operator.openshift.io
+  resources:
+  - imagecontentsourcepolicies
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ovn.openstack.org
   resources:

controllers/dataplane/openstackdataplanenodeset_controller.go

@@ -54,6 +54,8 @@ import (
 	dataplanev1 "github.com/openstack-k8s-operators/openstack-operator/apis/dataplane/v1beta1"
 	deployment "github.com/openstack-k8s-operators/openstack-operator/pkg/dataplane"
 	dataplaneutil "github.com/openstack-k8s-operators/openstack-operator/pkg/dataplane/util"
+
+	machineconfig "github.com/openshift/api/machineconfiguration/v1"
 )
 
 const (
@@ -120,6 +122,10 @@ func (r *OpenStackDataPlaneNodeSetReconciler) GetLogger(ctx context.Context) log
 // +kubebuilder:rbac:groups="image.openshift.io",resources=imagetags,verbs=get;list;watch
 // +kubebuilder:rbac:groups="image.openshift.io",resources=imagestreamtags,verbs=get;list;watch
 
+// RBAC for ImageContentSourcePolicy and MachineConfig
+// +kubebuilder:rbac:groups="operator.openshift.io",resources=imagecontentsourcepolicies,verbs=get;list
+// +kubebuilder:rbac:groups="machineconfiguration.openshift.io",resources=machineconfigs,verbs=get;list;watch
+
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 // TODO(user): Modify the Reconcile function to compare the state specified by
@@ -565,7 +571,8 @@ func checkDeployment(ctx context.Context, helper *helper.Helper,
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager(
-	ctx context.Context, mgr ctrl.Manager) error {
+	ctx context.Context, mgr ctrl.Manager,
+) error {
 	// index for ConfigMaps listed on ansibleVarsFrom
 	if err := mgr.GetFieldIndexer().IndexField(ctx,
 		&dataplanev1.OpenStackDataPlaneNodeSet{}, "spec.ansibleVarsFrom.ansible.configMaps",
@@ -639,9 +646,54 @@ func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager(
 			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})).
 		Watches(&openstackv1.OpenStackVersion{},
 			handler.EnqueueRequestsFromMapFunc(r.genericWatcherFn)).
+		Watches(&machineconfig.MachineConfig{},
+			handler.EnqueueRequestsFromMapFunc(r.machineConfigWatcherFn),
+			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})).
 		Complete(r)
 }
 
+// machineConfigWatcherFn - watches for changes to the registries MachineConfig resource and queues
+// a reconcile of each NodeSet if the MachineConfig is changed.
+func (r *OpenStackDataPlaneNodeSetReconciler) machineConfigWatcherFn(
+	ctx context.Context, obj client.Object,
+) []reconcile.Request {
+	Log := r.GetLogger(ctx)
+	nodeSets := &dataplanev1.OpenStackDataPlaneNodeSetList{}
+	kind := strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind)
+	const registryMachineConfigName string = "99-master-generated-registries"
+
+	mcNamespacedName := types.NamespacedName{
+		Name:      registryMachineConfigName,
+		Namespace: "",
+	}
+
+	if err := r.Get(ctx, mcNamespacedName, obj); err != nil {
+		Log.Error(err, fmt.Sprintf("Unable to retrieve MachingConfig %s", registryMachineConfigName))
+		return nil
+	}
+
+	listOpts := []client.ListOption{
+		client.InNamespace(obj.GetNamespace()),
+	}
+	if err := r.Client.List(ctx, nodeSets, listOpts...); err != nil {
+		Log.Error(err, "Unable to retrieve OpenStackDataPlaneNodeSetList")
+		return nil
+	}
+
+	requests := make([]reconcile.Request, 0, len(nodeSets.Items))
+	for _, nodeSet := range nodeSets.Items {
+		requests = append(requests, reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Namespace: obj.GetNamespace(),
+				Name:      nodeSet.Name,
+			},
+		})
+		Log.Info(fmt.Sprintf("reconcile loop for openstackdataplanenodeset %s triggered by %s %s",
+			nodeSet.Name, kind, obj.GetName()))
+	}
+	return requests
+}
+
 func (r *OpenStackDataPlaneNodeSetReconciler) secretWatcherFn(
 	ctx context.Context, obj client.Object,
 ) []reconcile.Request {

main.go

@@ -77,6 +77,8 @@ import (
 	dataplanev1 "github.com/openstack-k8s-operators/openstack-operator/apis/dataplane/v1beta1"
 
 	ocp_configv1 "github.com/openshift/api/config/v1"
+	machineconfig "github.com/openshift/api/machineconfiguration/v1"
+	ocp_image "github.com/openshift/api/operator/v1alpha1"
 	clientcontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/client"
 	corecontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/core"
 	dataplanecontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/dataplane"
@@ -120,6 +122,8 @@ func init() {
 	utilruntime.Must(certmgrv1.AddToScheme(scheme))
 	utilruntime.Must(barbicanv1.AddToScheme(scheme))
 	utilruntime.Must(ocp_configv1.AddToScheme(scheme))
+	utilruntime.Must(ocp_image.AddToScheme(scheme))
+	utilruntime.Must(machineconfig.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
 

pkg/dataplane/inventory.go

@@ -38,8 +38,7 @@ import (
 
 // getAnsibleVarsFrom gets ansible vars from ConfigMap/Secret
 func getAnsibleVarsFrom(ctx context.Context, helper *helper.Helper, namespace string, ansible *dataplanev1.AnsibleOpts) (map[string]interface{}, error) {
-
-	var result = make(map[string]interface{})
+	result := make(map[string]interface{})
 	var value interface{}
 
 	for _, dataSource := range ansible.AnsibleVarsFrom {
@@ -90,7 +89,8 @@ func GenerateNodeSetInventory(ctx context.Context, helper *helper.Helper,
 	instance *dataplanev1.OpenStackDataPlaneNodeSet,
 	allIPSets map[string]infranetworkv1.IPSet, dnsAddresses []string,
 	containerImages openstackv1.ContainerImages,
-	netServiceNetMap map[string]string) (string, error) {
+	netServiceNetMap map[string]string,
+) (string, error) {
 	inventory := ansible.MakeInventory()
 	nodeSetGroup := inventory.AddGroup(instance.Name)
 	groupVars, err := getAnsibleVarsFrom(ctx, helper, instance.Namespace, &instance.Spec.NodeTemplate.Ansible)
@@ -111,6 +111,23 @@ func GenerateNodeSetInventory(ctx context.Context, helper *helper.Helper,
 	// add the NodeSet name variable
 	nodeSetGroup.Vars["edpm_nodeset_name"] = instance.Name
 
+	isDisconnected, err := util.IsDisconnectedOCP(ctx, helper)
+	if err != nil {
+		return "", err
+	}
+
+	if isDisconnected {
+		registryConfig, err := util.GetMCRegistryConf(ctx, helper)
+		if err != nil {
+			return "", err
+		}
+		helper.GetLogger().Info("disconnected registry was identified via the ImageContentSourcePolicy. Using OCP registry.")
+
+		// Use OCP registries.conf for disconnected deployments
+		nodeSetGroup.Vars["edpm_podman_registries_conf"] = registryConfig
+		nodeSetGroup.Vars["edpm_podman_disconnected_ocp"] = isDisconnected
+	}
+
 	// add TLS ansible variable
 	nodeSetGroup.Vars["edpm_tls_certs_enabled"] = instance.Spec.TLSEnabled
 	if instance.Spec.Tags != nil {
@@ -205,7 +222,8 @@ func GenerateNodeSetInventory(ctx context.Context, helper *helper.Helper,
 // populateInventoryFromIPAM populates inventory from IPAM
 func populateInventoryFromIPAM(
 	ipSet *infranetworkv1.IPSet, host ansible.Host,
-	dnsAddresses []string, hostName string) {
+	dnsAddresses []string, hostName string,
+) {
 	var dnsSearchDomains []string
 	for _, res := range ipSet.Status.Reservation {
 		// Build the vars for ips/routes etc
@@ -243,8 +261,8 @@ func populateInventoryFromIPAM(
 // set group ansible vars from NodeTemplate
 func resolveGroupAnsibleVars(template *dataplanev1.NodeTemplate, group *ansible.Group,
 	containerImages openstackv1.ContainerImages,
-	netServiceNetMap map[string]string) error {
-
+	netServiceNetMap map[string]string,
+) error {
 	if template.Ansible.AnsibleUser != "" {
 		group.Vars["ansible_user"] = template.Ansible.AnsibleUser
 	}
@@ -320,8 +338,8 @@ func resolveGroupAnsibleVars(template *dataplanev1.NodeTemplate, group *ansible.
 
 // set host ansible vars from NodeSection
 func resolveHostAnsibleVars(node *dataplanev1.NodeSection,
-	host *ansible.Host, netServiceNetMap map[string]string) error {
-
+	host *ansible.Host, netServiceNetMap map[string]string,
+) error {
 	if node.Ansible.AnsibleUser != "" {
 		host.Vars["ansible_user"] = node.Ansible.AnsibleUser
 	}
@@ -344,13 +362,12 @@ func resolveHostAnsibleVars(node *dataplanev1.NodeSection,
 		host.Vars["network_servicenet_map"] = serviceNetMap
 	}
 	return nil
-
 }
 
 // unmarshal raw strings into an ansible vars dictionary
 func unmarshalAnsibleVars(ansibleVars map[string]json.RawMessage,
-	parsedVars map[string]interface{}) error {
-
+	parsedVars map[string]interface{},
+) error {
 	for key, val := range ansibleVars {
 		var v interface{}
 		err := yaml.Unmarshal(val, &v)
@@ -363,7 +380,8 @@ func unmarshalAnsibleVars(ansibleVars map[string]json.RawMessage,
 }
 
 func buildNetworkVars(networks []infranetworkv1.IPSetNetwork,
-	netServiceNetMap map[string]string) ([]string, map[string]string) {
+	netServiceNetMap map[string]string,
+) ([]string, map[string]string) {
 	serviceNetMap := make(map[string]string)
 	var nets []string
 	for _, network := range networks {

pkg/dataplane/util/image_registry.go

@@ -0,0 +1,131 @@
+package util
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	mc "github.com/openshift/api/machineconfiguration/v1"
+	ocpimage "github.com/openshift/api/operator/v1alpha1"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// machineConfigIgnition - holds the relevant parts of the ignition file that we need to create
+// the registries.conf on the dataplane nodes.
+type machineConfigIgnition struct {
+	Ignition struct {
+		Version string `json:"version"`
+	} `json:"ignition"`
+	Storage struct {
+		Files []struct {
+			Contents struct {
+				Compression string `json:"compression,omitempty"`
+				Source      string `json:"source"`
+			} `json:"contents"`
+			Mode      int    `json:"mode,omitempty"`
+			Overwrite bool   `json:"overwrite,omitempty"`
+			Path      string `json:"path"`
+		} `json:"files"`
+	} `json:"storage"`
+}
+
+// IsDisconnectedOCP - Will retrieve a ImageContentSourcePolicyList. If the list is not
+// empty, we can infer that the OCP cluster is a disconnected deployment.
+func IsDisconnectedOCP(ctx context.Context, helper *helper.Helper) (bool, error) {
+	icspList := ocpimage.ImageContentSourcePolicyList{}
+
+	listOpts := []client.ListOption{}
+	err := helper.GetClient().List(ctx, &icspList, listOpts...)
+	if err != nil {
+		return false, err
+	}
+
+	if len(icspList.Items) != 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// GetMCRegistryConf - will unmarshal the MachineConfig ignition file the machineConfigIgnition object.
+// This is then parsed and the base64 decoded string is returned.
+func GetMCRegistryConf(ctx context.Context, helper *helper.Helper) (string, error) {
+	var registriesConf string
+
+	masterMachineConfig, err := getMachineConfig(ctx, helper)
+	if err != nil {
+		return registriesConf, err
+	}
+
+	config := machineConfigIgnition{}
+	registriesConf, err = config.formatRegistriesConfString(&masterMachineConfig)
+	if err != nil {
+		return registriesConf, err
+	}
+
+	return registriesConf, nil
+}
+
+func (mci *machineConfigIgnition) removePrefixFromB64String() (string, error) {
+	const b64Prefix string = "data:text/plain;charset=utf-8;base64,"
+	if strings.HasPrefix(mci.Storage.Files[0].Contents.Source, b64Prefix) {
+		return mci.Storage.Files[0].Contents.Source[len(b64Prefix):], nil
+	}
+
+	return "", fmt.Errorf("no b64prefix found in MachineConfig")
+}
+
+func (mci *machineConfigIgnition) formatRegistriesConfString(machineConfig *mc.MachineConfig) (string, error) {
+	var (
+		err             error
+		rawConfigString string
+		configString    []byte
+	)
+
+	err = json.Unmarshal([]byte(machineConfig.Spec.Config.Raw), &mci)
+	if err != nil {
+		return "", err
+	}
+
+	rawConfigString, err = mci.removePrefixFromB64String()
+	if err != nil {
+		return "", err
+	}
+	configString, err = base64.StdEncoding.DecodeString(rawConfigString)
+	if err != nil {
+		return "", err
+	}
+
+	return string(configString), nil
+}
+
+func masterMachineConfigBuilder(machineConfigRegistries string) mc.MachineConfig {
+	return mc.MachineConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      machineConfigRegistries,
+			Namespace: "",
+		},
+	}
+}
+
+func getMachineConfig(ctx context.Context, helper *helper.Helper) (mc.MachineConfig, error) {
+	const machineConfigRegistries string = "99-master-generated-registries"
+
+	masterMachineConfig := masterMachineConfigBuilder(machineConfigRegistries)
+
+	err := helper.GetClient().Get(ctx,
+		types.NamespacedName{
+			Name: masterMachineConfig.Name, Namespace: masterMachineConfig.Namespace,
+		}, &masterMachineConfig)
+	if err != nil {
+		return masterMachineConfig, err
+	}
+
+	return masterMachineConfig, nil
+}