controller.go

package uiplugin

import (
	"context"
	"slices"
	"time"

	"github.com/go-logr/logr"
	configv1 "github.com/openshift/api/config/v1"
	osv1 "github.com/openshift/api/console/v1"
	operatorv1 "github.com/openshift/api/operator/v1"
	osRhobsv1 "github.com/rhobs/openshift-api/console/v1"
	osv1alpha1 "github.com/rhobs/openshift-api/console/v1alpha1"
	persesv1alpha2 "github.com/rhobs/perses-operator/api/v1alpha2"
	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metaerrors "k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/dynamic"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/builder"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/predicate"

	uiv1alpha1 "github.com/rhobs/observability-operator/pkg/apis/uiplugin/v1alpha1"
	"github.com/rhobs/observability-operator/pkg/reconciler"
)

type resourceManager struct {
	k8sClient        client.Client
	k8sDynamicClient dynamic.Interface
	scheme           *runtime.Scheme
	logger           logr.Logger
	controller       controller.Controller
	pluginConf       UIPluginsConfiguration
	clusterVersion   string
	apiReader        client.Reader
}

type UIPluginsConfiguration struct {
	Images             map[string]string
	ResourcesNamespace string
	TLSProfile         configv1.TLSProfileSpec
}

type Options struct {
	PluginsConf    UIPluginsConfiguration
	ClusterVersion string
}

const (
	AvailableReason         = "UIPluginAvailable"
	ReconciledReason        = "UIPluginReconciled"
	FailedToReconcileReason = "UIPluginFailedToReconcile"
	ReconciledMessage       = "Plugin reconciled successfully"
	NoReason                = "None"
)

// RBAC for managing UIPlugins
// +kubebuilder:rbac:groups=observability.openshift.io,resources=uiplugins,verbs=get;list;watch;create;update;delete;patch
// +kubebuilder:rbac:groups=observability.openshift.io,resources=uiplugins/status;uiplugins/finalizers,verbs=get;update

// RBAC for managing observability ui plugin objects
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=list;watch;create;update;delete;patch
//+kubebuilder:rbac:groups="",resources=serviceaccounts;services;configmaps,verbs=get;list;watch;create;update;patch;delete

// RBAC for managing Console CRs
// +kubebuilder:rbac:groups=operator.openshift.io,resources=consoles,verbs=get;patch;list;watch
// +kubebuilder:rbac:groups=console.openshift.io,resources=consoleplugins,verbs=get;list;watch;create;update;delete;patch

// RBAC for reading cluster version
// +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions,verbs=get;list;watch

// RBAC for reading TLS security profile from APIServer
// +kubebuilder:rbac:groups=config.openshift.io,resources=apiservers,verbs=get;list;watch

// RBAC for distributed tracing
// +kubebuilder:rbac:groups=tempo.grafana.com,resources=tempostacks;tempomonolithics,verbs=list

// RBAC for logging view plugin
// +kubebuilder:rbac:groups=loki.grafana.com,resources=application;infrastructure;audit,verbs=get
// +kubebuilder:rbac:groups=loki.grafana.com,resources=lokistacks,verbs=list;get

// RBAC for perses
// +kubebuilder:rbac:groups=perses.dev,resources=perses;persesdatasources;persesdashboards;persesglobaldatasources,verbs=get;list;watch;create;update;delete;patch
// +kubebuilder:rbac:groups=perses.dev,resources=perses/status;persesdatasources/status;persesglobaldatasources/status;persesdashboards/status,verbs=get;patch;update
// +kubebuilder:rbac:groups=perses.dev,resources=perses/finalizers;persesglobaldatasources/finalizers;persesdashboards/finalizers;persesdatasources/finalizers;persesdashboards/finalizers,verbs=update

// RBAC for delegating the use of SCC nonroot-v2 (for OpenShift >= 4.11) and nonroot (for OpenShift < 4.11) for Perses
//+kubebuilder:rbac:groups="security.openshift.io",resources=securitycontextconstraints,resourceNames=nonroot;nonroot-v2,verbs=use

// RBAC for korrel8r
//+kubebuilder:rbac:groups=apps,resources=daemonsets;deployments;replicasets;statefulsets,verbs=get;list;watch
//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings;clusterroles;clusterrolebindings,verbs=get;list;watch
//+kubebuilder:rbac:groups="",resources=configmaps;endpoints;events;namespaces;nodes;persistentvolumeclaims;persistentvolumes;pods;replicationcontrollers;secrets;serviceaccounts;services,verbs=get;list;watch
//+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch
//+kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;watch
//+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch
//+kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch
//+kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses;volumeattachments,verbs=get;list;watch
//+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies;ingresses,verbs=get;list;watch
//+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=create;delete;patch
//+kubebuilder:rbac:groups=loki.grafana.com,resources=application;infrastructure;audit;network,verbs=get
//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheuses/api,resourceNames=k8s,verbs=get;create;update
//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=alertmanagers/api,resourceNames=main,verbs=get;list

// RBAC for Korrel8r and Health Analyzer
//+kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create

// RBAC for Health Analyzer
//+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create
//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;create;update;patch;delete
//+kubebuilder:rbac:groups=config.openshift.io,resources=clusteroperators,verbs=get;list
//+kubebuilder:rbac:groups=machineconfiguration.openshift.io,resources=machineconfigpools,verbs=get;list
//+kubebuilder:rbac:groups=kubevirt.io,resources=kubevirts,verbs=get;list

const finalizerName = "uiplugin.observability.openshift.io/finalizer"

// RegisterWithManager registers the controller with Manager
func RegisterWithManager(mgr ctrl.Manager, opts Options) error {
	logger := ctrl.Log.WithName("observability-ui")

	dynamicClient, err := dynamic.NewForConfig(mgr.GetConfig())
	if err != nil {
		return err
	}

	rm := &resourceManager{
		k8sClient:        mgr.GetClient(),
		k8sDynamicClient: dynamicClient,
		scheme:           mgr.GetScheme(),
		logger:           logger,
		pluginConf:       opts.PluginsConf,
		clusterVersion:   opts.ClusterVersion,
		apiReader:        mgr.GetAPIReader(),
	}

	generationChanged := builder.WithPredicates(predicate.GenerationChangedPredicate{})

	ctrlBuilder := ctrl.NewControllerManagedBy(mgr).
		For(&uiv1alpha1.UIPlugin{}).
		Owns(&appsv1.Deployment{}, generationChanged).
		Owns(&v1.Service{}, generationChanged).
		Owns(&v1.ServiceAccount{}, generationChanged).
		Owns(&rbacv1.Role{}, generationChanged).
		Owns(&rbacv1.RoleBinding{}, generationChanged).
		Owns(&persesv1alpha2.Perses{}, generationChanged).
		Owns(&persesv1alpha2.PersesDashboard{}, generationChanged).
		Owns(&persesv1alpha2.PersesDatasource{}, generationChanged).
		Owns(&persesv1alpha2.PersesGlobalDatasource{}, generationChanged)

	if IsVersionAheadOrEqual(rm.clusterVersion, "v4.19") {
		ctrlBuilder.Owns(&osv1.ConsolePlugin{}, generationChanged)
	} else if IsVersionAheadOrEqual(rm.clusterVersion, "v4.17") {
		ctrlBuilder.Owns(&osRhobsv1.ConsolePlugin{}, generationChanged)
	} else {
		ctrlBuilder.Owns(&osv1alpha1.ConsolePlugin{}, generationChanged)
	}

	ctrl, err := ctrlBuilder.Build(rm)

	if err != nil {
		return err
	}
	rm.controller = ctrl

	return nil
}

func (rm resourceManager) consolePluginCapabilityEnabled(ctx context.Context, name types.NamespacedName, clusterVersion string) bool {
	var err error

	if IsVersionAheadOrEqual(clusterVersion, "v4.17") {
		consolePlugin := &osv1.ConsolePlugin{}
		err = rm.k8sClient.Get(ctx, name, consolePlugin)
	} else {
		legacyConsolePlugin := &osv1alpha1.ConsolePlugin{}
		err = rm.k8sClient.Get(ctx, name, legacyConsolePlugin)
	}

	return err == nil || !metaerrors.IsNoMatchError(err)
}

func (rm resourceManager) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	logger := rm.logger.WithValues("plugin", req.NamespacedName)

	if !rm.consolePluginCapabilityEnabled(ctx, req.NamespacedName, rm.clusterVersion) {
		logger.Info("Cluster console plugin not supported or not accessible. Skipping observability UI plugin reconciliation")
		return ctrl.Result{}, nil
	}

	logger.Info("Reconciling observability UI plugin")

	plugin, err := rm.getUIPlugin(ctx, req)
	if err != nil {
		// retry since some error has occurred
		return ctrl.Result{}, err
	}

	if plugin == nil {
		// no such obs ui plugin, so stop here
		return ctrl.Result{}, nil
	}

	// Check if the plugin is being deleted
	if !plugin.ObjectMeta.DeletionTimestamp.IsZero() {
		logger.V(6).Info("deregistering plugin from the console")
		if err := rm.deregisterPluginFromConsole(ctx, pluginTypeToConsoleName[plugin.Spec.Type]); err != nil {
			logger.V(3).Info("best-effort console deregistration failed during deletion", "error", err)
		}

		if err := rm.removeLegacyFinalizer(ctx, plugin); err != nil {
			return ctrl.Result{}, err
		}

		logger.V(6).Info("skipping reconcile since object is already scheduled for deletion")
		return ctrl.Result{}, nil
	}

	if err := rm.removeLegacyFinalizer(ctx, plugin); err != nil {
		return ctrl.Result{}, err
	}

	compatibilityInfo, err := lookupImageAndFeatures(plugin.Spec.Type, rm.clusterVersion)
	if err != nil {
		return rm.updateStatus(ctx, req, plugin, err), err
	}

	if plugin.Annotations == nil {
		plugin.Annotations = map[string]string{}
		plugin.Annotations["observability.openshift.io/api-support"] = string(compatibilityInfo.SupportLevel)

		if err := rm.k8sClient.Update(ctx, plugin); err != nil {
			return ctrl.Result{}, err
		}
		// Upon changes to the uiplugin, reconciliation will happen anyways, so just return early
		// and let the reconciliation handle the further changes
		return ctrl.Result{}, nil
	} else if plugin.Annotations["observability.openshift.io/api-support"] != string(compatibilityInfo.SupportLevel) {
		plugin.Annotations["observability.openshift.io/api-support"] = string(compatibilityInfo.SupportLevel)

		if err := rm.k8sClient.Update(ctx, plugin); err != nil {
			return ctrl.Result{}, err
		}
		return ctrl.Result{}, nil
	}

	pluginInfo, pluginInfoErr := PluginInfoBuilder(ctx, rm.k8sClient, rm.k8sDynamicClient, plugin, rm.pluginConf, compatibilityInfo, rm.clusterVersion, rm.logger)

	if pluginInfo != nil {
		reconcilers := pluginComponentReconcilers(plugin, *pluginInfo, rm.clusterVersion, rm.logger)
		for _, reconciler := range reconcilers {
			err := reconciler.Reconcile(ctx, rm.k8sClient, rm.scheme)
			// handle creation / updation errors that can happen due to a stale cache by
			// retrying after some time.
			if apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) {
				logger.V(8).Info("skipping reconcile error", "err", err)
				return ctrl.Result{RequeueAfter: 2 * time.Second}, nil
			}
			if err != nil {
				return rm.updateStatus(ctx, req, plugin, err), err
			}
		}
		if pluginInfo.AreMonitoringFeatsDisabled {
			// prevents double rendering of monitoring console-tabs
			if err := rm.deregisterPluginFromConsole(ctx, pluginTypeToConsoleName[plugin.Spec.Type]); err != nil {
				return rm.updateStatus(ctx, req, plugin, err), err
			}
		}
	}

	if pluginInfoErr != nil {
		// If features are disabled allow pluginComponentReconcilers to remove uiplugin-related components before status update
		return rm.updateStatus(ctx, req, plugin, pluginInfoErr), pluginInfoErr
	}

	if err := rm.registerPluginWithConsole(ctx, pluginInfo); err != nil {
		return rm.updateStatus(ctx, req, plugin, err), err
	}

	return rm.updateStatus(ctx, req, plugin, nil), nil
}

func (rm resourceManager) updateStatus(ctx context.Context, req ctrl.Request, pl *uiv1alpha1.UIPlugin, recError error) ctrl.Result {
	logger := rm.logger.WithValues("plugin", req.NamespacedName)

	var changed bool

	if recError != nil {
		changed = meta.SetStatusCondition(&pl.Status.Conditions, metav1.Condition{
			Type:               string(uiv1alpha1.DegradedCondition),
			Status:             metav1.ConditionTrue,
			Reason:             FailedToReconcileReason,
			Message:            recError.Error(),
			ObservedGeneration: pl.Generation,
		}) || changed
		changed = meta.SetStatusCondition(&pl.Status.Conditions, metav1.Condition{
			Type:               string(uiv1alpha1.ReconciledCondition),
			Status:             metav1.ConditionFalse,
			Reason:             FailedToReconcileReason,
			Message:            recError.Error(),
			ObservedGeneration: pl.Generation,
		}) || changed

		changed = meta.SetStatusCondition(&pl.Status.Conditions, metav1.Condition{
			Type:               string(uiv1alpha1.AvailableCondition),
			Status:             metav1.ConditionFalse,
			Reason:             FailedToReconcileReason,
			ObservedGeneration: pl.Generation,
		}) || changed
	} else {
		changed = meta.SetStatusCondition(&pl.Status.Conditions, metav1.Condition{
			Type:               string(uiv1alpha1.ReconciledCondition),
			Status:             metav1.ConditionTrue,
			Reason:             ReconciledReason,
			Message:            ReconciledMessage,
			ObservedGeneration: pl.Generation,
		}) || changed
		changed = meta.SetStatusCondition(&pl.Status.Conditions, metav1.Condition{
			Type:               string(uiv1alpha1.DegradedCondition),
			Status:             metav1.ConditionFalse,
			Reason:             ReconciledReason,
			ObservedGeneration: pl.Generation,
		}) || changed

		changed = meta.SetStatusCondition(&pl.Status.Conditions, metav1.Condition{
			Type:               string(uiv1alpha1.AvailableCondition),
			Status:             metav1.ConditionTrue,
			Reason:             AvailableReason,
			ObservedGeneration: pl.Generation,
		}) || changed
	}

	if !changed {
		return ctrl.Result{}
	}

	err := rm.k8sClient.Status().Update(ctx, pl)
	if err != nil {
		logger.Info("Failed to update status", "err", err)
		return ctrl.Result{RequeueAfter: 2 * time.Second}
	}

	return ctrl.Result{}
}

func (rm resourceManager) registerPluginWithConsole(ctx context.Context, pluginInfo *UIPluginInfo) error {
	cluster := &operatorv1.Console{}
	if err := rm.apiReader.Get(ctx, client.ObjectKey{Name: "cluster"}, cluster); err != nil {
		return err
	}

	if slices.Contains(cluster.Spec.Plugins, pluginInfo.ConsoleName) {
		return nil
	}

	clusterPlugins := append(cluster.Spec.Plugins, pluginInfo.ConsoleName)

	// Register the plugin with the console
	cluster.Spec.Plugins = clusterPlugins

	if err := reconciler.NewMerger(cluster, pluginInfo.ConsoleName).Reconcile(ctx, rm.k8sClient, rm.scheme); err != nil {
		return err
	}

	return nil
}

func (rm resourceManager) deregisterPluginFromConsole(ctx context.Context, pluginConsoleName string) error {
	cluster := &operatorv1.Console{}
	if err := rm.apiReader.Get(ctx, client.ObjectKey{Name: "cluster"}, cluster); err != nil {
		return err
	}

	if !slices.Contains(cluster.Spec.Plugins, pluginConsoleName) {
		return nil
	}

	clusterPlugins := slices.DeleteFunc(cluster.Spec.Plugins, func(currentPluginName string) bool {
		return currentPluginName == pluginConsoleName
	})

	// Deregister the plugin from the console
	cluster.Spec.Plugins = clusterPlugins

	if err := reconciler.NewMerger(cluster, pluginConsoleName).Reconcile(ctx, rm.k8sClient, rm.scheme); err != nil {
		return err
	}

	return nil
}

func (rm resourceManager) removeLegacyFinalizer(ctx context.Context, plugin *uiv1alpha1.UIPlugin) error {
	if !controllerutil.ContainsFinalizer(plugin, finalizerName) {
		return nil
	}
	patch := client.MergeFrom(plugin.DeepCopy())
	controllerutil.RemoveFinalizer(plugin, finalizerName)
	if err := rm.k8sClient.Patch(ctx, plugin, patch); err != nil {
		if apierrors.IsNotFound(err) {
			return nil
		}
		return err
	}
	return nil
}

func (rm resourceManager) getUIPlugin(ctx context.Context, req ctrl.Request) (*uiv1alpha1.UIPlugin, error) {
	logger := rm.logger.WithValues("plugin", req.NamespacedName)

	plugin := uiv1alpha1.UIPlugin{}

	if err := rm.k8sClient.Get(ctx, req.NamespacedName, &plugin); err != nil {
		if apierrors.IsNotFound(err) {
			logger.V(3).Info("stack could not be found; may be marked for deletion")
			return nil, nil
		}
		return nil, err
	}

	return &plugin, nil
}