virtualmcpserver_controller.go

// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
// SPDX-License-Identifier: Apache-2.0

// Package controllers contains the reconciliation logic for the VirtualMCPServer custom resource.
// It handles the creation, update, and deletion of Virtual MCP Servers in Kubernetes.
package controllers

import (
	"context"
	"crypto/rand"
	"encoding/base64"
	stderrors "errors"
	"fmt"
	"maps"
	"reflect"
	"strings"
	"time"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/tools/events"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"

	mcpv1alpha1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1alpha1"
	ctrlutil "github.com/stacklok/toolhive/cmd/thv-operator/pkg/controllerutil"
	"github.com/stacklok/toolhive/cmd/thv-operator/pkg/kubernetes/rbac"
	"github.com/stacklok/toolhive/cmd/thv-operator/pkg/runconfig/configmap/checksum"
	"github.com/stacklok/toolhive/cmd/thv-operator/pkg/virtualmcpserverstatus"
	vmcptypes "github.com/stacklok/toolhive/pkg/vmcp"
	"github.com/stacklok/toolhive/pkg/vmcp/auth/converters"
	authtypes "github.com/stacklok/toolhive/pkg/vmcp/auth/types"
	vmcpconfig "github.com/stacklok/toolhive/pkg/vmcp/config"
	"github.com/stacklok/toolhive/pkg/vmcp/workloads"
)

const (
	// OutgoingAuthSourceDiscovered indicates that auth configs should be automatically discovered from MCPServers
	OutgoingAuthSourceDiscovered = "discovered"
	// OutgoingAuthSourceInline indicates that auth configs should be explicitly specified
	OutgoingAuthSourceInline = "inline"

	// Auth config error context constants
	authContextDefault          = "default"
	authContextBackendPrefix    = "backend:"
	authContextDiscoveredPrefix = "discovered:"
)

// AuthConfigError represents a single auth config conversion failure.
// It captures context about which auth config failed and why, allowing the controller
// to continue in degraded mode while exposing the failure via status conditions.
//
// Context patterns:
//   - "default": Default auth config (OutgoingAuth.Default)
//   - "backend:<name>": Inline backend-specific config (OutgoingAuth.Backends[name])
//   - "discovered:<name>": Discovered from MCPServer/MCPRemoteProxy ExternalAuthConfigRef
type AuthConfigError struct {
	// Context describes where the error occurred: "default", "backend:<name>", or "discovered:<name>"
	Context string
	// BackendName is the backend name (empty for default auth config)
	BackendName string
	// Error is the underlying error that occurred during conversion
	Error error
}

// SpecValidationError represents a spec validation failure that the user must fix.
// Unlike transient errors, these should NOT trigger requeue — the controller sets
// a status condition and waits for the user to update the spec.
type SpecValidationError struct {
	Message string
}

func (e *SpecValidationError) Error() string {
	return e.Message
}

// VirtualMCPServerReconciler reconciles a VirtualMCPServer object
//
// Resource Cleanup Strategy:
// VirtualMCPServer does NOT use finalizers because all managed resources have owner references
// set via controllerutil.SetControllerReference. Kubernetes automatically cascade-deletes
// owned resources when the VirtualMCPServer is deleted. Managed resources include:
//   - Deployment (owned)
//   - Service (owned)
//   - ConfigMap for vmcp config (owned)
//   - ServiceAccount, Role, RoleBinding via rbac.Client (owned)
//
// This differs from MCPServer which uses finalizers to explicitly delete resources that
// may not have owner references (StatefulSet, headless Service, RunConfig ConfigMap).
type VirtualMCPServerReconciler struct {
	client.Client
	Scheme           *runtime.Scheme
	Recorder         events.EventRecorder
	PlatformDetector *ctrlutil.SharedPlatformDetector
}

// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=virtualmcpservers,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=virtualmcpservers/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=mcpgroups,verbs=get;list;watch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=mcpservers,verbs=get;list;watch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=mcpexternalauthconfigs,verbs=get;list;watch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=mcptoolconfigs,verbs=get;list;watch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=virtualmcpcompositetooldefinitions,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=configmaps,verbs=create;delete;get;list;patch;update;watch
// +kubebuilder:rbac:groups="",resources=services,verbs=create;delete;get;list;patch;update;watch
// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=create;delete;get;list;patch;update;watch
// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=create;delete;get;list;patch;update;watch
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
// +kubebuilder:rbac:groups="",resources=secrets,verbs=create;get;list;watch
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;delete;get;list;patch;update;watch
// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=create;delete;get;list;patch;update;watch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=mcpoidcconfigs,verbs=get;list;watch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=mcpoidcconfigs/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=embeddingservers,verbs=get;list;watch
// +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=embeddingservers/status,verbs=get

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *VirtualMCPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	ctxLogger := log.FromContext(ctx)

	// Fetch the VirtualMCPServer instance
	vmcp := &mcpv1alpha1.VirtualMCPServer{}
	err := r.Get(ctx, req.NamespacedName, vmcp)
	if err != nil {
		if errors.IsNotFound(err) {
			ctxLogger.Info("VirtualMCPServer resource not found. Ignoring since object must be deleted")
			return ctrl.Result{}, nil
		}
		ctxLogger.Error(err, "Failed to get VirtualMCPServer")
		return ctrl.Result{}, err
	}

	// Create status manager for batched updates
	statusManager := virtualmcpserverstatus.NewStatusManager(vmcp)

	// Run all pre-reconciliation validations.
	// Returns (true, nil) to continue, (false, nil) when validation failed but
	// should not requeue (user must fix spec), or (false, err) for transient errors
	// that should trigger requeue.
	if cont, err := r.runValidations(ctx, vmcp, statusManager); err != nil {
		return ctrl.Result{}, err
	} else if !cont {
		return ctrl.Result{}, nil
	}

	// Validate MCPOIDCConfigRef if referenced (before resource creation).
	// handleOIDCConfig is a no-op when OIDCConfigRef is nil.
	if err := r.handleOIDCConfig(ctx, vmcp, statusManager); err != nil {
		if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
			ctxLogger.Error(applyErr, "Failed to apply status updates after MCPOIDCConfig validation error")
		}
		return ctrl.Result{}, err
	}

	// Ensure all resources
	if result, err := r.ensureAllResources(ctx, vmcp, statusManager); err != nil {
		// Apply status changes before returning error
		if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
			ctxLogger.Error(applyErr, "Failed to apply status updates after resource reconciliation error")
		}
		return ctrl.Result{}, err
	} else if result.RequeueAfter > 0 {
		// Apply status changes before returning requeue (e.g., waiting for EmbeddingServer)
		if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
			ctxLogger.Error(applyErr, "Failed to apply status updates before requeue")
		}
		return result, nil
	}

	// Backend discovery and health reporting is now delegated to the vMCP runtime (StatusReporter).
	// The runtime reports status.discoveredBackends, status.backendCount, backend health, and
	// BackendsDiscovered condition based on actual MCP connectivity and health checks.
	//
	// Controller responsibilities (infrastructure-only):
	// - RBAC (ServiceAccount, Role, RoleBinding)
	// - Deployment, Service, ConfigMap
	// - GroupRef validation
	// - Infrastructure conditions (DeploymentReady, ServiceReady)
	// - status.URL
	//
	// Runtime responsibilities (via StatusReporter with VMCP_NAME/VMCP_NAMESPACE env vars):
	// - Backend discovery from MCPGroup
	// - Backend health monitoring (ready/degraded/unavailable)
	// - status.Phase (Ready/Degraded/Failed)
	// - status.discoveredBackends with health status
	// - status.backendCount
	// - BackendsDiscovered condition

	// Fetch the latest version before updating status to ensure we use the current Generation
	latestVMCP := &mcpv1alpha1.VirtualMCPServer{}
	if err := r.Get(ctx, types.NamespacedName{
		Name:      vmcp.Name,
		Namespace: vmcp.Namespace,
	}, latestVMCP); err != nil {
		ctxLogger.Error(err, "Failed to get latest VirtualMCPServer before status update")
		return ctrl.Result{}, err
	}

	// Update status based on pod health using the latest Generation
	if err := r.updateVirtualMCPServerStatus(ctx, latestVMCP, statusManager); err != nil {
		ctxLogger.Error(err, "Failed to update VirtualMCPServer status")
		return ctrl.Result{}, err
	}

	// Apply all collected status changes in a single batch update
	if err := r.applyStatusUpdates(ctx, latestVMCP, statusManager); err != nil {
		ctxLogger.Error(err, "Failed to apply final status updates")
		return ctrl.Result{}, err
	}

	// Reconciliation complete - rely on event-driven reconciliation
	// Kubernetes will automatically trigger reconcile when:
	// - VirtualMCPServer spec changes
	// - Referenced resources (MCPGroup, Secrets) change
	// - Owned resources (Deployment, Service) status changes
	// - vmcp pods emit events about backend health
	return ctrl.Result{}, nil
}

// validateSpec validates the VirtualMCPServer spec and updates status on error.
// Returns an error if validation fails, which signals the caller to stop reconciliation.
func (r *VirtualMCPServerReconciler) validateSpec(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	ctxLogger := log.FromContext(ctx)

	if err := vmcp.Validate(); err != nil {
		ctxLogger.Error(err, "VirtualMCPServer spec validation failed")
		statusManager.SetObservedGeneration(vmcp.Generation)
		statusManager.SetCondition("Valid", "ValidationFailed", err.Error(), metav1.ConditionFalse)
		if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
			ctxLogger.Error(applyErr, "Failed to apply status updates after validation error")
		}
		return err
	}

	// Validation succeeded - set Valid=True condition
	statusManager.SetObservedGeneration(vmcp.Generation)
	statusManager.SetCondition("Valid", "ValidationSucceeded", "Spec validation passed", metav1.ConditionTrue)

	return nil
}

// applyStatusUpdates applies all collected status changes in a single batch update.
// This implements the StatusCollector pattern to reduce API calls and prevent update conflicts.
func (r *VirtualMCPServerReconciler) applyStatusUpdates(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	ctxLogger := log.FromContext(ctx)

	// Fetch the latest version to avoid conflicts
	latest := &mcpv1alpha1.VirtualMCPServer{}
	if err := r.Get(ctx, types.NamespacedName{
		Name:      vmcp.Name,
		Namespace: vmcp.Namespace,
	}, latest); err != nil {
		return fmt.Errorf("failed to get latest VirtualMCPServer: %w", err)
	}

	// Apply collected changes to the latest status
	hasUpdates := statusManager.UpdateStatus(ctx, &latest.Status)

	// Only update if there are changes
	if hasUpdates {
		if err := r.Status().Update(ctx, latest); err != nil {
			// Handle conflicts by returning error to trigger requeue
			if errors.IsConflict(err) {
				ctxLogger.V(1).Info("Conflict updating status, will requeue")
				return err
			}
			return fmt.Errorf("failed to update VirtualMCPServer status: %w", err)
		}
		ctxLogger.V(1).Info("Successfully applied batched status updates")
	}

	return nil
}

// runValidations runs all pre-reconciliation validations (PodTemplateSpec, GroupRef,
// CompositeToolRefs, EmbeddingServerRef, AuthServerConfig).
// Returns (true, nil) to continue reconciliation.
// Returns (false, nil) for spec validation errors that should NOT trigger requeue
// (user must fix the spec; next reconciliation is triggered by spec changes).
// Returns (false, error) for transient errors that should trigger requeue.
func (r *VirtualMCPServerReconciler) runValidations(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) (bool, error) {
	ctxLogger := log.FromContext(ctx)

	// Validate spec configuration early (schema-level validation from types.go).
	// Don't requeue on validation errors — user must fix spec.
	if err := r.validateSpec(ctx, vmcp, statusManager); err != nil {
		return false, nil
	}

	// Validate PodTemplateSpec early - before other validations.
	// Don't requeue — user must fix the PodTemplateSpec.
	if !r.validateAndUpdatePodTemplateStatus(ctx, vmcp, statusManager) {
		if err := r.applyStatusUpdates(ctx, vmcp, statusManager); err != nil {
			ctxLogger.Error(err, "Failed to apply status updates after PodTemplateSpec validation error")
		}
		return false, nil
	}

	// Validate GroupRef
	if err := r.validateGroupRef(ctx, vmcp, statusManager); err != nil {
		if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
			ctxLogger.Error(applyErr, "Failed to apply status updates after GroupRef validation error")
		}
		return false, err
	}

	// Validate CompositeToolRefs
	if err := r.validateCompositeToolRefs(ctx, vmcp, statusManager); err != nil {
		if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
			ctxLogger.Error(applyErr, "Failed to apply status updates after CompositeToolRefs validation error")
		}
		return false, err
	}

	// Validate EmbeddingServerRef (when using reference mode)
	if vmcp.Spec.EmbeddingServerRef != nil {
		if err := r.validateEmbeddingServerRef(ctx, vmcp, statusManager); err != nil {
			if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
				ctxLogger.Error(applyErr, "Failed to apply status updates after EmbeddingServerRef validation error")
			}
			return false, err
		}
	}

	// Validate inline AuthServerConfig (when specified).
	if vmcp.Spec.AuthServerConfig != nil {
		if err := r.validateAuthServerConfig(vmcp, statusManager); err != nil {
			if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
				ctxLogger.Error(applyErr, "Failed to apply status updates after AuthServerConfig validation error")
			}
			return false, nil
		}
	} else {
		// Remove stale condition if AuthServerConfig was previously set then removed.
		statusManager.RemoveConditionsWithPrefix(mcpv1alpha1.ConditionTypeAuthServerConfigValidated, []string{})
	}

	// Advisory: warn when replicas > 1 but session storage is not Redis-backed.
	r.validateSessionStorageForReplicas(vmcp, statusManager)

	return true, nil
}

// validateSessionStorageForReplicas emits a SessionStorageWarning condition when
// replicas > 1 but session storage is not configured with a Redis backend.
// Reconciliation continues regardless; this is advisory only.
func (*VirtualMCPServerReconciler) validateSessionStorageForReplicas(
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) {
	if vmcp.Spec.Replicas != nil && *vmcp.Spec.Replicas > 1 {
		if vmcp.Spec.SessionStorage == nil || vmcp.Spec.SessionStorage.Provider != mcpv1alpha1.SessionStorageProviderRedis {
			statusManager.SetCondition(
				mcpv1alpha1.ConditionSessionStorageWarning,
				mcpv1alpha1.ConditionReasonSessionStorageMissing,
				"replicas > 1 but sessionStorage.provider is not redis; sessions are not shared across replicas",
				metav1.ConditionTrue,
			)
		} else {
			statusManager.SetCondition(
				mcpv1alpha1.ConditionSessionStorageWarning,
				mcpv1alpha1.ConditionReasonSessionStorageConfigured,
				"Redis session storage is configured",
				metav1.ConditionFalse,
			)
		}
	} else {
		statusManager.SetCondition(
			mcpv1alpha1.ConditionSessionStorageWarning,
			mcpv1alpha1.ConditionReasonSessionStorageNotApplicable,
			"session storage warning is not active",
			metav1.ConditionFalse,
		)
	}
}

// validateAuthServerConfig validates inline AuthServerConfig and sets the
// AuthServerConfigValidated condition. Returns an error when validation fails
// (caller should NOT requeue — user must fix the spec).
func (*VirtualMCPServerReconciler) validateAuthServerConfig(
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	cfg := vmcp.Spec.AuthServerConfig

	if cfg.Issuer == "" {
		message := "spec.authServerConfig.issuer is required"
		statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhaseFailed)
		statusManager.SetMessage(message)
		statusManager.SetAuthServerConfigValidatedCondition(
			mcpv1alpha1.ConditionReasonAuthServerConfigInvalid,
			message,
			metav1.ConditionFalse,
		)
		statusManager.SetObservedGeneration(vmcp.Generation)
		return fmt.Errorf("%s", message)
	}

	if len(cfg.UpstreamProviders) == 0 {
		message := "spec.authServerConfig.upstreamProviders is required"
		statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhaseFailed)
		statusManager.SetMessage(message)
		statusManager.SetAuthServerConfigValidatedCondition(
			mcpv1alpha1.ConditionReasonAuthServerConfigInvalid,
			message,
			metav1.ConditionFalse,
		)
		statusManager.SetObservedGeneration(vmcp.Generation)
		return fmt.Errorf("%s", message)
	}

	// AuthServerConfig is valid
	statusManager.SetAuthServerConfigValidatedCondition(
		mcpv1alpha1.ConditionReasonAuthServerConfigValid,
		"AuthServerConfig is valid",
		metav1.ConditionTrue,
	)
	statusManager.SetObservedGeneration(vmcp.Generation)

	return nil
}

// handleSpecValidationError checks whether err is a SpecValidationError (user must fix the spec).
// If so, it applies the already-set status conditions and returns nil (no requeue).
// Otherwise it returns the original error unchanged for normal requeue handling.
func (r *VirtualMCPServerReconciler) handleSpecValidationError(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
	err error,
) error {
	var specErr *SpecValidationError
	if !stderrors.As(err, &specErr) {
		return err
	}
	ctxLogger := log.FromContext(ctx)
	if applyErr := r.applyStatusUpdates(ctx, vmcp, statusManager); applyErr != nil {
		ctxLogger.Error(applyErr, "Failed to apply status updates after spec validation error")
		return applyErr
	}
	return nil
}

// validateGroupRef validates that the referenced MCPGroup exists and is ready
func (r *VirtualMCPServerReconciler) validateGroupRef(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	ctxLogger := log.FromContext(ctx)

	// Validate GroupRef exists
	mcpGroup := &mcpv1alpha1.MCPGroup{}
	err := r.Get(ctx, types.NamespacedName{
		Name:      vmcp.Spec.Config.Group,
		Namespace: vmcp.Namespace,
	}, mcpGroup)

	if errors.IsNotFound(err) {
		message := fmt.Sprintf("Referenced MCPGroup %s not found", vmcp.Spec.Config.Group)
		statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhaseFailed)
		statusManager.SetMessage(message)
		statusManager.SetGroupRefValidatedCondition(
			mcpv1alpha1.ConditionReasonVirtualMCPServerGroupRefNotFound,
			message,
			metav1.ConditionFalse,
		)
		statusManager.SetObservedGeneration(vmcp.Generation)
		return err
	} else if err != nil {
		ctxLogger.Error(err, "Failed to get MCPGroup")
		return err
	}

	// Check if MCPGroup is ready
	if mcpGroup.Status.Phase != mcpv1alpha1.MCPGroupPhaseReady {
		message := fmt.Sprintf("Referenced MCPGroup %s is not ready (phase: %s)",
			vmcp.Spec.Config.Group, mcpGroup.Status.Phase)
		statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhasePending)
		statusManager.SetMessage(message)
		statusManager.SetGroupRefValidatedCondition(
			mcpv1alpha1.ConditionReasonVirtualMCPServerGroupRefNotReady,
			message,
			metav1.ConditionFalse,
		)
		statusManager.SetObservedGeneration(vmcp.Generation)
		// Requeue to check again later
		return fmt.Errorf("MCPGroup %s is not ready", vmcp.Spec.Config.Group)
	}

	// GroupRef is valid and ready
	statusManager.SetGroupRefValidatedCondition(
		mcpv1alpha1.ConditionReasonVirtualMCPServerGroupRefValid,
		fmt.Sprintf("MCPGroup %s is valid and ready", vmcp.Spec.Config.Group),
		metav1.ConditionTrue,
	)
	statusManager.SetObservedGeneration(vmcp.Generation)

	return nil
}

// validateCompositeToolRefs validates that all referenced VirtualMCPCompositeToolDefinition resources exist
func (r *VirtualMCPServerReconciler) validateCompositeToolRefs(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	ctxLogger := log.FromContext(ctx)

	// If no composite tool refs, nothing to validate
	if len(vmcp.Spec.Config.CompositeToolRefs) == 0 {
		// Set condition to indicate validation passed (no refs to validate)
		statusManager.SetObservedGeneration(vmcp.Generation)
		statusManager.SetCompositeToolRefsValidatedCondition(
			mcpv1alpha1.ConditionReasonCompositeToolRefsValid,
			"No composite tool references to validate",
			metav1.ConditionTrue,
		)
		return nil
	}

	// Validate each referenced composite tool definition exists
	for i := range vmcp.Spec.Config.CompositeToolRefs {
		ref := &vmcp.Spec.Config.CompositeToolRefs[i]
		compositeToolDef := &mcpv1alpha1.VirtualMCPCompositeToolDefinition{}
		err := r.Get(ctx, types.NamespacedName{
			Name:      ref.Name,
			Namespace: vmcp.Namespace,
		}, compositeToolDef)

		if errors.IsNotFound(err) {
			message := fmt.Sprintf("Referenced VirtualMCPCompositeToolDefinition %s not found", ref.Name)
			statusManager.SetObservedGeneration(vmcp.Generation)
			statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhaseFailed)
			statusManager.SetMessage(message)
			statusManager.SetCompositeToolRefsValidatedCondition(
				mcpv1alpha1.ConditionReasonCompositeToolRefNotFound,
				message,
				metav1.ConditionFalse,
			)
			return err
		} else if err != nil {
			ctxLogger.Error(err, "Failed to get VirtualMCPCompositeToolDefinition", "name", ref.Name)
			return err
		}

		// Check that the composite tool definition is validated and valid
		if compositeToolDef.Status.ValidationStatus == mcpv1alpha1.ValidationStatusInvalid {
			message := fmt.Sprintf("Referenced VirtualMCPCompositeToolDefinition %s is invalid", ref.Name)
			if len(compositeToolDef.Status.ValidationErrors) > 0 {
				message = fmt.Sprintf("%s: %s", message, strings.Join(compositeToolDef.Status.ValidationErrors, "; "))
			}
			statusManager.SetObservedGeneration(vmcp.Generation)
			statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhaseFailed)
			statusManager.SetMessage(message)
			statusManager.SetCompositeToolRefsValidatedCondition(
				mcpv1alpha1.ConditionReasonCompositeToolRefInvalid,
				message,
				metav1.ConditionFalse,
			)
			return fmt.Errorf("referenced VirtualMCPCompositeToolDefinition %s is invalid", ref.Name)
		}

		// If ValidationStatus is Unknown, we still allow it (validation might be in progress)
		// but log a warning
		if compositeToolDef.Status.ValidationStatus == mcpv1alpha1.ValidationStatusUnknown {
			ctxLogger.V(1).Info("Referenced composite tool definition validation status is Unknown, proceeding",
				"name", ref.Name, "namespace", vmcp.Namespace)
		}
	}

	// All composite tool refs are valid
	statusManager.SetObservedGeneration(vmcp.Generation)
	statusManager.SetCompositeToolRefsValidatedCondition(
		mcpv1alpha1.ConditionReasonCompositeToolRefsValid,
		fmt.Sprintf("All %d composite tool references are valid", len(vmcp.Spec.Config.CompositeToolRefs)),
		metav1.ConditionTrue,
	)

	return nil
}

// validateAndUpdatePodTemplateStatus validates the PodTemplateSpec and uses StatusManager to collect
// status changes. Returns true if validation passes, false otherwise.
// The caller is responsible for applying status updates via applyStatusUpdates().
func (r *VirtualMCPServerReconciler) validateAndUpdatePodTemplateStatus(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) bool {
	ctxLogger := log.FromContext(ctx)

	// Only validate if PodTemplateSpec is provided
	if vmcp.Spec.PodTemplateSpec == nil || vmcp.Spec.PodTemplateSpec.Raw == nil {
		// No PodTemplateSpec provided, validation passes
		return true
	}

	_, err := ctrlutil.NewPodTemplateSpecBuilder(vmcp.Spec.PodTemplateSpec, "vmcp")
	if err != nil {
		// Record event for invalid PodTemplateSpec
		if r.Recorder != nil {
			r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "InvalidPodTemplateSpec", "ValidatePodTemplateSpec",
				"Failed to parse PodTemplateSpec: %v. Deployment blocked until PodTemplateSpec is fixed.", err)
		}

		// Use StatusManager to collect status changes
		statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhaseFailed)
		statusManager.SetMessage(fmt.Sprintf("Invalid PodTemplateSpec: %v", err))
		statusManager.SetCondition(
			mcpv1alpha1.ConditionTypeVirtualMCPServerPodTemplateSpecValid,
			mcpv1alpha1.ConditionReasonVirtualMCPServerPodTemplateSpecInvalid,
			fmt.Sprintf("Failed to parse PodTemplateSpec: %v. Deployment blocked until fixed.", err),
			metav1.ConditionFalse,
		)
		statusManager.SetObservedGeneration(vmcp.Generation)

		ctxLogger.Error(err, "PodTemplateSpec validation failed")
		return false
	}

	// Use StatusManager to collect status changes for valid PodTemplateSpec
	statusManager.SetCondition(
		mcpv1alpha1.ConditionTypeVirtualMCPServerPodTemplateSpecValid,
		mcpv1alpha1.ConditionReasonVirtualMCPServerPodTemplateSpecValid,
		"PodTemplateSpec is valid",
		metav1.ConditionTrue,
	)
	statusManager.SetObservedGeneration(vmcp.Generation)

	return true
}

// ensureAllResources ensures all Kubernetes resources for the VirtualMCPServer.
// Returns a ctrl.Result with RequeueAfter when the controller should retry later
// (e.g., waiting for EmbeddingServer readiness), and an error for failures.
func (r *VirtualMCPServerReconciler) ensureAllResources(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) (ctrl.Result, error) {
	ctxLogger := log.FromContext(ctx)

	// Validate secret references before creating resources.
	// This catches configuration errors early, providing faster feedback than waiting for pod startup failures.
	if err := r.ensureAuthSecretsValid(ctx, vmcp, statusManager); err != nil {
		return ctrl.Result{}, err
	}

	// Check EmbeddingServer readiness before proceeding to Deployment.
	// RequeueAfter provides a safety net in case the Watches() events
	// are missed (e.g., EmbeddingServer controller not running).
	esURL, err := r.isEmbeddingServerReady(ctx, vmcp)
	if err != nil {
		return ctrl.Result{}, err
	}
	// EmbeddingServer is configured but not yet ready — requeue
	if esURL == nil && vmcp.Spec.EmbeddingServerRef != nil {
		statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhasePending)
		statusManager.SetMessage("Waiting for EmbeddingServer to become ready")
		statusManager.SetEmbeddingServerReadyCondition(
			mcpv1alpha1.ConditionReasonEmbeddingServerNotReady,
			"EmbeddingServer is not yet ready",
			metav1.ConditionFalse,
		)
		return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
	}

	// If an embedding server is configured and ready, set the condition
	if esURL != nil {
		statusManager.SetEmbeddingServerReadyCondition(
			mcpv1alpha1.ConditionReasonEmbeddingServerReady,
			"EmbeddingServer is ready",
			metav1.ConditionTrue,
		)
	}

	// List workloads once and pass to functions that need them
	// This ensures consistency - all functions use the same workload list
	// rather than listing at different times which could yield different results
	workloadDiscoverer := workloads.NewK8SDiscovererWithClient(r.Client, vmcp.Namespace)
	workloadNames, err := workloadDiscoverer.ListWorkloadsInGroup(ctx, vmcp.Spec.Config.Group)
	if err != nil {
		ctxLogger.Error(err, "Failed to list workloads in group")
		return ctrl.Result{}, fmt.Errorf("failed to list workloads in group: %w", err)
	}

	// Ensure RBAC resources
	if err := r.ensureRBACResources(ctx, vmcp); err != nil {
		ctxLogger.Error(err, "Failed to ensure RBAC resources")
		return ctrl.Result{}, err
	}

	// Ensure HMAC secret for session token binding (Session Management V2)
	if err := r.ensureHMACSecret(ctx, vmcp); err != nil {
		ctxLogger.Error(err, "Failed to ensure HMAC secret")
		return ctrl.Result{}, err
	}

	// Ensure vmcp Config ConfigMap.
	// handleSpecValidationError converts SpecValidationError to nil (no requeue)
	// after applying status conditions, while passing through transient errors.
	if specValidationErr := r.ensureVmcpConfigConfigMap(ctx, vmcp, workloadNames, statusManager); specValidationErr != nil {
		if err := r.handleSpecValidationError(ctx, vmcp, statusManager, specValidationErr); err != nil {
			ctxLogger.Error(err, "Failed to ensure vmcp Config ConfigMap")
			return ctrl.Result{}, err
		}
		// SpecValidationError: status applied, stop reconciliation without requeue.
		// Do not proceed to ensureDeployment — the ConfigMap was not created/updated.
		return ctrl.Result{}, nil
	}

	// Ensure Deployment
	if result, err := r.ensureDeployment(ctx, vmcp, workloadNames); err != nil {
		return ctrl.Result{}, err
	} else if result.RequeueAfter > 0 {
		return result, nil
	}

	// Ensure Service
	if result, err := r.ensureService(ctx, vmcp); err != nil {
		return ctrl.Result{}, err
	} else if result.RequeueAfter > 0 {
		return result, nil
	}

	// Update service URL in status
	r.ensureServiceURL(vmcp, statusManager)
	return ctrl.Result{}, nil
}

// ensureAuthSecretsValid validates secret references and sets the AuthConfigured condition.
func (r *VirtualMCPServerReconciler) ensureAuthSecretsValid(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	if err := r.validateSecretReferences(ctx, vmcp); err != nil {
		ctxLogger := log.FromContext(ctx)
		ctxLogger.Error(err, "Secret validation failed")
		statusManager.SetAuthConfiguredCondition(
			mcpv1alpha1.ConditionReasonAuthInvalid,
			fmt.Sprintf("Authentication configuration is invalid: %v", err),
			metav1.ConditionFalse,
		)
		statusManager.SetObservedGeneration(vmcp.Generation)
		if r.Recorder != nil {
			r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "SecretValidationFailed", "ValidateSecrets",
				"Secret validation failed: %v", err)
		}
		return err
	}

	statusManager.SetAuthConfiguredCondition(
		mcpv1alpha1.ConditionReasonAuthValid,
		"Authentication configuration is valid",
		metav1.ConditionTrue,
	)
	statusManager.SetObservedGeneration(vmcp.Generation)
	return nil
}

// ensureRBACResources ensures RBAC resources for VirtualMCPServer.
// RBAC resources are created in all modes (discovered and inline) to support:
// - Backend discovery (discovered mode only)
// - Status reporting via K8sReporter (all modes)
//
// When a custom ServiceAccount is provided, RBAC creation is skipped.
//
// Uses the RBAC client (pkg/kubernetes/rbac) which creates or updates RBAC resources
// automatically during operator upgrades.
func (r *VirtualMCPServerReconciler) ensureRBACResources(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
) error {
	// If a service account is specified, we don't need to create one
	if vmcp.Spec.ServiceAccount != nil {
		return nil
	}

	rbacClient := rbac.NewClient(r.Client, r.Scheme)
	serviceAccountName := vmcpServiceAccountName(vmcp.Name)

	// Select RBAC rules based on outgoing auth mode
	// - inline mode: Minimal permissions (read own spec + update status)
	// - discovered mode: Full permissions (read secrets, configmaps, MCP resources + update status)
	rules := func() []rbacv1.PolicyRule {
		if outgoingAuthSource(vmcp) == OutgoingAuthSourceInline {
			// inline mode uses minimal permissions (no secret/configmap access)
			return vmcpInlineRBACRules
		}
		// discovered mode (default)
		return vmcpDiscoveredRBACRules
	}()

	// Ensure Role with appropriate permissions based on mode
	_, err := rbacClient.EnsureRBACResources(ctx, rbac.EnsureRBACResourcesParams{
		Name:      serviceAccountName,
		Namespace: vmcp.Namespace,
		Rules:     rules,
		Owner:     vmcp,
	})
	return err
}

// ensureHMACSecret ensures the HMAC secret exists for session token binding.
// This secret is required when Session Management V2 is enabled.
// The secret is automatically generated with a cryptographically secure random value.
//
// The secret follows this naming pattern: {vmcp-name}-hmac-secret
// and contains a single key: hmac-secret with a 32-byte base64-encoded random value.
func (r *VirtualMCPServerReconciler) ensureHMACSecret(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
) error {
	ctxLogger := log.FromContext(ctx)

	secretName := fmt.Sprintf("%s-hmac-secret", vmcp.Name)
	secret := &corev1.Secret{}
	err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: vmcp.Namespace}, secret)

	if errors.IsNotFound(err) {
		// Generate a cryptographically secure 32-byte HMAC secret
		hmacSecret, err := generateHMACSecret()
		if err != nil {
			ctxLogger.Error(err, "Failed to generate HMAC secret")
			if r.Recorder != nil {
				r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "HMACSecretGenerationFailed", "GenerateHMACSecret",
					"Failed to generate HMAC secret: %v", err)
			}
			return fmt.Errorf("failed to generate HMAC secret: %w", err)
		}

		newSecret := &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      secretName,
				Namespace: vmcp.Namespace,
				Labels: map[string]string{
					"app.kubernetes.io/name":       "virtualmcpserver",
					"app.kubernetes.io/instance":   vmcp.Name,
					"app.kubernetes.io/component":  "session-security",
					"app.kubernetes.io/managed-by": "toolhive-operator",
				},
				Annotations: map[string]string{
					"toolhive.stacklok.dev/purpose": "hmac-secret-for-session-token-binding",
				},
			},
			Type: corev1.SecretTypeOpaque,
			Data: map[string][]byte{
				"hmac-secret": []byte(hmacSecret),
			},
		}

		// Set VirtualMCPServer as owner so secret is automatically deleted when VMCP is deleted
		if err := controllerutil.SetControllerReference(vmcp, newSecret, r.Scheme); err != nil {
			ctxLogger.Error(err, "Failed to set controller reference for HMAC secret")
			return fmt.Errorf("failed to set controller reference: %w", err)
		}

		ctxLogger.Info("Creating HMAC secret for session token binding", "Secret.Name", secretName)
		if err := r.Create(ctx, newSecret); err != nil {
			ctxLogger.Error(err, "Failed to create HMAC secret")
			if r.Recorder != nil {
				r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "HMACSecretCreationFailed", "CreateHMACSecret",
					"Failed to create HMAC secret: %v", err)
			}
			return fmt.Errorf("failed to create HMAC secret: %w", err)
		}

		// Record success event
		if r.Recorder != nil {
			r.Recorder.Eventf(vmcp, nil, corev1.EventTypeNormal, "HMACSecretCreated", "CreateHMACSecret",
				"HMAC secret created for session token binding")
		}
		return nil
	} else if err != nil {
		ctxLogger.Error(err, "Failed to get HMAC secret")
		return fmt.Errorf("failed to get HMAC secret: %w", err)
	}

	// Secret exists - validate ownership and structure before accepting it
	if err := r.validateHMACSecret(ctx, vmcp, secret); err != nil {
		ctxLogger.Error(err, "Existing HMAC secret is invalid", "Secret.Name", secretName)
		if r.Recorder != nil {
			r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "HMACSecretValidationFailed", "ValidateHMACSecret",
				"Existing HMAC secret validation failed: %v", err)
		}
		return fmt.Errorf("existing HMAC secret validation failed: %w", err)
	}

	return nil
}

// validateHMACSecret validates that an existing HMAC secret has the correct ownership,
// structure, and content. This prevents accepting stale, malformed, or attacker-controlled
// secrets that could weaken session token signing or cause pod startup failures.
func (*VirtualMCPServerReconciler) validateHMACSecret(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	secret *corev1.Secret,
) error {
	ctxLogger := log.FromContext(ctx)

	// Verify the secret is owned by this VirtualMCPServer
	// This prevents accepting secrets created by other actors
	isOwned := false
	for _, ownerRef := range secret.OwnerReferences {
		if ownerRef.UID == vmcp.UID &&
			ownerRef.Kind == "VirtualMCPServer" &&
			ownerRef.Name == vmcp.Name {
			isOwned = true
			break
		}
	}
	if !isOwned {
		return fmt.Errorf("secret is not owned by VirtualMCPServer %s/%s", vmcp.Namespace, vmcp.Name)
	}

	// Verify the hmac-secret key exists
	hmacSecretData, exists := secret.Data["hmac-secret"]
	if !exists {
		return fmt.Errorf("secret missing required 'hmac-secret' key")
	}

	// Verify it's valid base64 and decodes to exactly 32 bytes
	hmacSecretBase64 := string(hmacSecretData)
	if hmacSecretBase64 == "" {
		return fmt.Errorf("hmac-secret is empty")
	}

	decoded, err := base64.StdEncoding.DecodeString(hmacSecretBase64)
	if err != nil {
		return fmt.Errorf("hmac-secret is not valid base64: %w", err)
	}

	if len(decoded) != 32 {
		return fmt.Errorf("hmac-secret must be exactly 32 bytes, got %d bytes", len(decoded))
	}

	// Verify it's not all zeros (would indicate a weak/predictable key)
	allZeros := true
	for _, b := range decoded {
		if b != 0 {
			allZeros = false
			break
		}
	}
	if allZeros {
		return fmt.Errorf("hmac-secret is all zeros (weak key)")
	}

	ctxLogger.V(1).Info("HMAC secret validation passed", "Secret.Name", secret.Name)
	return nil
}

// getVmcpConfigChecksum fetches the vmcp Config ConfigMap checksum annotation.
// This is used to trigger deployment rollouts when the configuration changes.
//
// Note: VirtualMCPServer uses a custom ConfigMap naming pattern ("{name}-vmcp-config")
// instead of the standard "{name}-runconfig" pattern, so it cannot use the shared
// checksum.RunConfigChecksumFetcher. However, it follows the same validation logic
// and uses the same annotation constant for consistency.
func (r *VirtualMCPServerReconciler) getVmcpConfigChecksum(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
) (string, error) {
	if vmcp == nil {
		return "", fmt.Errorf("vmcp cannot be nil")
	}

	configMapName := vmcpConfigMapName(vmcp.Name)
	configMap := &corev1.ConfigMap{}
	err := r.Get(ctx, types.NamespacedName{
		Name:      configMapName,
		Namespace: vmcp.Namespace,
	}, configMap)

	if err != nil {
		// Preserve error type for IsNotFound checks
		return "", fmt.Errorf("failed to get vmcp Config ConfigMap %s/%s: %w",
			vmcp.Namespace, configMapName, err)
	}

	// Use the standard checksum annotation constant for consistency
	checksumValue, ok := configMap.Annotations[checksum.ContentChecksumAnnotation]
	if !ok {
		return "", fmt.Errorf("vmcp Config ConfigMap %s/%s missing %s annotation",
			vmcp.Namespace, configMapName, checksum.ContentChecksumAnnotation)
	}

	if checksumValue == "" {
		return "", fmt.Errorf("vmcp Config ConfigMap %s/%s has empty %s annotation",
			vmcp.Namespace, configMapName, checksum.ContentChecksumAnnotation)
	}

	return checksumValue, nil
}

// ensureDeployment ensures the Deployment exists and is up to date
//
//nolint:unparam // ctrl.Result needed for ConfigMap not found case (RequeueAfter)
func (r *VirtualMCPServerReconciler) ensureDeployment(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	typedWorkloads []workloads.TypedWorkload,
) (ctrl.Result, error) {
	ctxLogger := log.FromContext(ctx)

	// Fetch vmcp Config ConfigMap checksum to include in pod template annotations
	vmcpConfigChecksum, err := r.getVmcpConfigChecksum(ctx, vmcp)
	if err != nil {
		if errors.IsNotFound(err) {
			ctxLogger.Info("vmcp Config ConfigMap not found yet, will retry",
				"vmcp", vmcp.Name, "namespace", vmcp.Namespace)
			return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
		}
		ctxLogger.Error(err, "Failed to get vmcp Config checksum")
		return ctrl.Result{}, err
	}

	deployment := &appsv1.Deployment{}
	err = r.Get(ctx, types.NamespacedName{Name: vmcp.Name, Namespace: vmcp.Namespace}, deployment)

	if errors.IsNotFound(err) {
		dep := r.deploymentForVirtualMCPServer(ctx, vmcp, vmcpConfigChecksum, typedWorkloads)
		if dep == nil {
			return ctrl.Result{}, fmt.Errorf("failed to create Deployment object")
		}
		ctxLogger.Info("Creating a new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
		if err := r.Create(ctx, dep); err != nil {
			ctxLogger.Error(err, "Failed to create new Deployment")
			// Record event for deployment creation failure
			if r.Recorder != nil {
				r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "DeploymentCreationFailed", "CreateDeployment",
					"Failed to create Deployment: %v", err)
			}
			return ctrl.Result{}, err
		}
		// Record event for successful deployment creation
		if r.Recorder != nil {
			r.Recorder.Eventf(vmcp, nil, corev1.EventTypeNormal, "DeploymentCreated", "CreateDeployment",
				"Deployment created successfully")
		}
		// Return empty result to continue with rest of reconciliation (Service, status update, etc.)
		// Kubernetes will automatically requeue when Deployment status changes
		return ctrl.Result{}, nil
	} else if err != nil {
		ctxLogger.Error(err, "Failed to get Deployment")
		return ctrl.Result{}, err
	}

	// Deployment exists - check if it needs to be updated
	// deploymentNeedsUpdate performs a detailed comparison to avoid unnecessary updates
	if r.deploymentNeedsUpdate(ctx, deployment, vmcp, vmcpConfigChecksum, typedWorkloads) {
		newDeployment := r.deploymentForVirtualMCPServer(ctx, vmcp, vmcpConfigChecksum, typedWorkloads)
		if newDeployment == nil {
			return ctrl.Result{}, fmt.Errorf("failed to create updated Deployment object")
		}

		// Selective field update strategy:
		// - Update Spec.Template: Contains container spec, volumes, pod metadata (triggers rollout)
		// - Update Labels: For label selectors and queries
		// - Update Annotations: For metadata and tooling
		// - Sync Spec.Replicas when spec.replicas is non-nil (operator authoritative)
		// - Preserve Spec.Replicas when spec.replicas is nil (HPA or external controller manages scaling)
		// - Preserve ResourceVersion, UID: Required for optimistic concurrency control
		//
		// Note: If update conflicts occur due to concurrent modifications, the reconcile
		// loop will retry automatically. Kubernetes' optimistic locking prevents data loss.
		deployment.Spec.Template = newDeployment.Spec.Template
		deployment.Labels = newDeployment.Labels
		deployment.Annotations = ctrlutil.MergeAnnotations(newDeployment.Annotations, deployment.Annotations)
		if newDeployment.Spec.Replicas != nil {
			deployment.Spec.Replicas = newDeployment.Spec.Replicas
		}

		ctxLogger.Info("Updating Deployment", "Deployment.Namespace", deployment.Namespace, "Deployment.Name", deployment.Name)
		if err := r.Update(ctx, deployment); err != nil {
			ctxLogger.Error(err, "Failed to update Deployment")
			// Record event for deployment update failure
			if r.Recorder != nil {
				r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "DeploymentUpdateFailed", "UpdateDeployment",
					"Failed to update Deployment: %v", err)
			}
			// Return error to trigger reconcile retry (handles transient failures and conflicts)
			return ctrl.Result{}, err
		}
		// Record event for successful deployment update (config change triggers rollout)
		if r.Recorder != nil {
			r.Recorder.Eventf(vmcp, nil, corev1.EventTypeNormal, "DeploymentUpdated", "UpdateDeployment",
				"Deployment updated, rolling out new configuration")
		}
		// Return empty result to continue with rest of reconciliation
		// Deployment rollout will be monitored when Kubernetes triggers subsequent reconciles
		return ctrl.Result{}, nil
	}

	return ctrl.Result{}, nil
}

// ensureService ensures the Service exists and is up to date
//
//nolint:unparam // ctrl.Result kept for consistency with ensureDeployment signature
func (r *VirtualMCPServerReconciler) ensureService(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
) (ctrl.Result, error) {
	ctxLogger := log.FromContext(ctx)

	serviceName := vmcpServiceName(vmcp.Name)
	service := &corev1.Service{}
	err := r.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: vmcp.Namespace}, service)

	if errors.IsNotFound(err) {
		svc := r.serviceForVirtualMCPServer(ctx, vmcp)
		if svc == nil {
			return ctrl.Result{}, fmt.Errorf("failed to create Service object")
		}
		ctxLogger.Info("Creating a new Service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name)
		if err := r.Create(ctx, svc); err != nil {
			ctxLogger.Error(err, "Failed to create new Service")
			// Record event for service creation failure
			if r.Recorder != nil {
				r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "ServiceCreationFailed", "CreateService",
					"Failed to create Service: %v", err)
			}
			return ctrl.Result{}, err
		}
		// Record event for successful service creation
		if r.Recorder != nil {
			r.Recorder.Eventf(vmcp, nil, corev1.EventTypeNormal, "ServiceCreated", "CreateService",
				"Service %s created successfully", serviceName)
		}
		// Return empty result to continue with rest of reconciliation
		return ctrl.Result{}, nil
	} else if err != nil {
		ctxLogger.Error(err, "Failed to get Service")
		return ctrl.Result{}, err
	}

	// Service exists - check if it needs to be updated
	// serviceNeedsUpdate compares ports, type, labels, and annotations
	if r.serviceNeedsUpdate(service, vmcp) {
		newService := r.serviceForVirtualMCPServer(ctx, vmcp)
		if newService == nil {
			return ctrl.Result{}, fmt.Errorf("failed to create updated Service object")
		}

		// Selective field update strategy for Service:
		// - Update Spec.Ports: Modify exposed ports
		// - Update Spec.Type: Change service type (ClusterIP, NodePort, LoadBalancer)
		// - Update Labels: For selectors and queries
		// - Update Annotations: For metadata and tooling
		// - Preserve Spec.ClusterIP: Immutable field, cannot be changed
		// - Preserve Spec.HealthCheckNodePort: Set by cloud provider for LoadBalancer
		// - Preserve ResourceVersion, UID: Required for optimistic concurrency control
		service.Spec.Ports = newService.Spec.Ports
		service.Spec.Type = newService.Spec.Type
		service.Spec.SessionAffinity = newService.Spec.SessionAffinity
		service.Labels = newService.Labels
		service.Annotations = newService.Annotations

		ctxLogger.Info("Updating Service", "Service.Namespace", service.Namespace, "Service.Name", service.Name)
		if err := r.Update(ctx, service); err != nil {
			ctxLogger.Error(err, "Failed to update Service")
			return ctrl.Result{}, err
		}
		// Return empty result to continue with rest of reconciliation
		return ctrl.Result{}, nil
	}

	return ctrl.Result{}, nil
}

// ensureServiceURL ensures the service URL is set in the status
func (*VirtualMCPServerReconciler) ensureServiceURL(
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) {
	if vmcp.Status.URL == "" {
		url := createVmcpServiceURL(vmcp.Name, vmcp.Namespace, vmcpDefaultPort)
		statusManager.SetURL(url)
	}
}

// deploymentNeedsUpdate checks if the deployment needs to be updated
func (r *VirtualMCPServerReconciler) deploymentNeedsUpdate(
	ctx context.Context,
	deployment *appsv1.Deployment,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	vmcpConfigChecksum string,
	typedWorkloads []workloads.TypedWorkload,
) bool {
	if deployment == nil || vmcp == nil {
		return true
	}

	if len(deployment.Spec.Template.Spec.Containers) == 0 {
		return true
	}

	if r.containerNeedsUpdate(ctx, deployment, vmcp, typedWorkloads) {
		return true
	}

	if r.deploymentMetadataNeedsUpdate(deployment, vmcp) {
		return true
	}

	if r.podTemplateMetadataNeedsUpdate(deployment, vmcp, vmcpConfigChecksum) {
		return true
	}

	if r.podTemplateSpecNeedsUpdate(ctx, deployment, vmcp, typedWorkloads) {
		return true
	}

	// Check if spec.replicas has changed. Only compare when spec.replicas is non-nil;
	// nil means hands-off mode (HPA or external controller manages replicas) and the live count is authoritative.
	if vmcp.Spec.Replicas != nil {
		if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != *vmcp.Spec.Replicas {
			return true
		}
	}

	return false
}

// containerNeedsUpdate checks if the container specification has changed
func (r *VirtualMCPServerReconciler) containerNeedsUpdate(
	ctx context.Context,
	deployment *appsv1.Deployment,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	typedWorkloads []workloads.TypedWorkload,
) bool {
	if deployment == nil || vmcp == nil || len(deployment.Spec.Template.Spec.Containers) == 0 {
		return true
	}

	container := deployment.Spec.Template.Spec.Containers[0]

	// Check if vmcp image has changed
	expectedImage := getVmcpImage()
	if container.Image != expectedImage {
		return true
	}

	// Check if port has changed
	if len(container.Ports) > 0 && container.Ports[0].ContainerPort != vmcpDefaultPort {
		return true
	}

	// Check if container args have changed (includes --debug flag from logLevel)
	expectedArgs := r.buildContainerArgsForVmcp(vmcp)
	if !reflect.DeepEqual(container.Args, expectedArgs) {
		return true
	}

	// Check if environment variables have changed
	expectedEnv, err := r.buildEnvVarsForVmcp(ctx, vmcp, typedWorkloads)
	if err != nil {
		return true // Trigger update to surface the error
	}
	if !reflect.DeepEqual(container.Env, expectedEnv) {
		return true
	}

	// Check if service account has changed
	expectedServiceAccountName := r.serviceAccountNameForVmcp(vmcp)
	currentServiceAccountName := deployment.Spec.Template.Spec.ServiceAccountName
	return currentServiceAccountName != expectedServiceAccountName
}

// deploymentMetadataNeedsUpdate checks if deployment-level metadata has changed
func (*VirtualMCPServerReconciler) deploymentMetadataNeedsUpdate(
	deployment *appsv1.Deployment,
	vmcp *mcpv1alpha1.VirtualMCPServer,
) bool {
	if deployment == nil || vmcp == nil {
		return true
	}

	expectedLabels := labelsForVirtualMCPServer(vmcp.Name)
	expectedAnnotations := make(map[string]string)

	// TODO: Add support for ResourceOverrides if needed in the future

	// Check that all expected labels are present with correct values
	// (Allows Kubernetes-managed labels to exist without triggering updates)
	for key, expectedValue := range expectedLabels {
		if actualValue, exists := deployment.Labels[key]; !exists || actualValue != expectedValue {
			return true
		}
	}

	// Check that all expected annotations are present with correct values
	// (Allows Kubernetes-managed annotations like deployment.kubernetes.io/revision to exist)
	for key, expectedValue := range expectedAnnotations {
		if actualValue, exists := deployment.Annotations[key]; !exists || actualValue != expectedValue {
			return true
		}
	}

	return false
}

// podTemplateMetadataNeedsUpdate checks if pod template metadata has changed
func (r *VirtualMCPServerReconciler) podTemplateMetadataNeedsUpdate(
	deployment *appsv1.Deployment,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	vmcpConfigChecksum string,
) bool {
	if deployment == nil || vmcp == nil {
		return true
	}

	expectedPodTemplateLabels, expectedPodTemplateAnnotations := r.buildPodTemplateMetadata(
		labelsForVirtualMCPServer(vmcp.Name), vmcp, vmcpConfigChecksum,
	)

	if !maps.Equal(deployment.Spec.Template.Labels, expectedPodTemplateLabels) {
		return true
	}

	if !maps.Equal(deployment.Spec.Template.Annotations, expectedPodTemplateAnnotations) {
		return true
	}

	return false
}

// podTemplateSpecNeedsUpdate checks if the user-provided PodTemplateSpec has changed.
// Instead of comparing full rendered templates (which always differ due to Kubernetes-defaulted
// fields like terminationGracePeriodSeconds, dnsPolicy, etc.), this compares a SHA256 hash of
// the raw PodTemplateSpec input stored as a deployment annotation.
func (*VirtualMCPServerReconciler) podTemplateSpecNeedsUpdate(
	ctx context.Context,
	deployment *appsv1.Deployment,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	_ []workloads.TypedWorkload,
) bool {
	if deployment == nil || vmcp == nil {
		return true
	}

	// If no PodTemplateSpec is provided, update is only needed if one was previously applied
	if vmcp.Spec.PodTemplateSpec == nil || vmcp.Spec.PodTemplateSpec.Raw == nil {
		_, hadPrevious := deployment.Annotations[podTemplateSpecHashAnnotation]
		return hadPrevious
	}

	// Compare hash of the raw PodTemplateSpec input against the stored annotation.
	// Avoids comparing full rendered templates which always differ due to
	// Kubernetes-defaulted fields (terminationGracePeriodSeconds, dnsPolicy, etc.).
	// Uses HashRawJSON to ensure deterministic hashing regardless of JSON field ordering.
	expectedHash, err := checksum.HashRawJSON(vmcp.Spec.PodTemplateSpec.Raw)
	if err != nil {
		// If we can't hash, assume update is needed
		log.FromContext(ctx).Error(err, "Failed to hash PodTemplateSpec, assuming update needed")
		return true
	}
	return deployment.Annotations[podTemplateSpecHashAnnotation] != expectedHash
}

// serviceNeedsUpdate checks if the service needs to be updated
func (*VirtualMCPServerReconciler) serviceNeedsUpdate(
	service *corev1.Service,
	vmcp *mcpv1alpha1.VirtualMCPServer,
) bool {
	if service == nil || vmcp == nil {
		return true
	}

	// Check if port has changed
	if len(service.Spec.Ports) > 0 && service.Spec.Ports[0].Port != vmcpDefaultPort {
		return true
	}

	// Check if service type has changed
	expectedServiceType := corev1.ServiceTypeClusterIP
	if vmcp.Spec.ServiceType != "" {
		expectedServiceType = corev1.ServiceType(vmcp.Spec.ServiceType)
	}
	if service.Spec.Type != expectedServiceType {
		return true
	}

	// Check if session affinity has drifted from spec
	expectedAffinity := func() corev1.ServiceAffinity {
		if vmcp.Spec.SessionAffinity != "" {
			return corev1.ServiceAffinity(vmcp.Spec.SessionAffinity)
		}
		return corev1.ServiceAffinityClientIP
	}()
	if service.Spec.SessionAffinity != expectedAffinity {
		return true
	}

	// Check if service metadata has changed
	expectedLabels := labelsForVirtualMCPServer(vmcp.Name)
	expectedAnnotations := make(map[string]string)

	// TODO: Add support for ResourceOverrides if needed in the future

	if !maps.Equal(service.Labels, expectedLabels) {
		return true
	}

	if !maps.Equal(service.Annotations, expectedAnnotations) {
		return true
	}

	return false
}

// updateVirtualMCPServerStatus updates the status of the VirtualMCPServer based on pod and backend health.
//
// Status Update Pattern and Conflict Handling:
//
// This controller follows the status update pattern established by MCPGroup controller in this codebase.
// Status updates occur at multiple points during reconciliation:
//
//  1. Early Error States: Status updates happen immediately when validation or discovery fails
//     (e.g., GroupRef not found, GroupRef not ready, backend discovery failed)
//
// 2. Mid-Reconciliation: Status fields like URL are set when resources are created
//
// 3. Final Status: This function performs the comprehensive final status update by:
//   - Listing all pods for the deployment
//   - Checking backend health status
//   - Computing overall phase (Ready, Degraded, Pending, Failed)
//   - Setting appropriate conditions
//   - Updating ObservedGeneration to track which spec version was reconciled
//
// Conflict Handling Strategy:
// All Status().Update() calls now include explicit conflict detection using errors.IsConflict().
// When conflicts occur:
// - The error is returned to the controller runtime
// - Controller runtime automatically requeues the reconciliation
// - Next reconcile loop will GET the latest resource version and retry
//
// This implements Kubernetes' optimistic concurrency control pattern and prevents lost updates
// when multiple controllers or processes modify the same resource. The MCPGroup controller
// demonstrates this pattern is the established best practice in this codebase.
//
// Why Not a Separate Status Reconciler?
// This codebase does not use separate status-only reconcile loops. Status and spec reconciliation
// happen in the same loop, which is appropriate for this use case because:
// - Status depends on spec reconciliation (need deployment/service to exist first)
// - Status updates are not frequent enough to warrant separate reconciliation
// - Single reconcile loop is simpler and matches existing codebase patterns

// statusDecision encapsulates the status update decision to reduce branching and repetition
type statusDecision struct {
	phase          mcpv1alpha1.VirtualMCPServerPhase
	message        string
	reason         string
	conditionMsg   string
	conditionState metav1.ConditionStatus
}

// countBackendHealth counts ready and unhealthy backends
func countBackendHealth(ctx context.Context, backends []mcpv1alpha1.DiscoveredBackend) (ready, unhealthy int) {
	ctxLogger := log.FromContext(ctx)

	for _, backend := range backends {
		switch backend.Status {
		case mcpv1alpha1.BackendStatusReady:
			ready++
		case mcpv1alpha1.BackendStatusUnavailable,
			mcpv1alpha1.BackendStatusDegraded,
			mcpv1alpha1.BackendStatusUnknown:
			unhealthy++
		default:
			ctxLogger.V(1).Info("Unexpected backend status, treating as unhealthy",
				"backend", backend.Name, "status", backend.Status)
			unhealthy++
		}
	}
	return ready, unhealthy
}

// determineStatusFromBackends evaluates backend health to determine status
func (*VirtualMCPServerReconciler) determineStatusFromBackends(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
) statusDecision {
	ctxLogger := log.FromContext(ctx)

	ready, unhealthy := countBackendHealth(ctx, vmcp.Status.DiscoveredBackends)
	total := ready + unhealthy

	// All backends unhealthy
	if ready == 0 && unhealthy > 0 {
		return statusDecision{
			phase:          mcpv1alpha1.VirtualMCPServerPhaseDegraded,
			message:        fmt.Sprintf("Virtual MCP server is running but all %d backends are unhealthy", unhealthy),
			reason:         "BackendsUnavailable",
			conditionMsg:   "All backends are unhealthy",
			conditionState: metav1.ConditionFalse,
		}
	}

	// Some backends unhealthy
	if unhealthy > 0 {
		return statusDecision{
			phase:          mcpv1alpha1.VirtualMCPServerPhaseDegraded,
			message:        fmt.Sprintf("Virtual MCP server is running with %d/%d backends available", ready, total),
			reason:         "BackendsDegraded",
			conditionMsg:   "Some backends are unhealthy",
			conditionState: metav1.ConditionFalse,
		}
	}

	// All backends ready
	if ready > 0 {
		return statusDecision{
			phase:          mcpv1alpha1.VirtualMCPServerPhaseReady,
			message:        "Virtual MCP server is running",
			reason:         "DeploymentReady",
			conditionMsg:   "Deployment is ready",
			conditionState: metav1.ConditionTrue,
		}
	}

	// Edge case: backends exist but none counted
	ctxLogger.V(1).Info("No backends were counted, treating as degraded",
		"discoveredBackendsCount", len(vmcp.Status.DiscoveredBackends))
	return statusDecision{
		phase:          mcpv1alpha1.VirtualMCPServerPhaseDegraded,
		message:        "Virtual MCP server is running but backend status cannot be determined",
		reason:         "BackendsUnknown",
		conditionMsg:   "Backend status unknown",
		conditionState: metav1.ConditionFalse,
	}
}

// determineStatusFromPods determines the appropriate status based on pod states.
// The 'ready' parameter counts pods that have passed their readiness probes (PodReady condition is True),
// not just pods in Running phase. This ensures the VirtualMCPServer is only marked Ready when
// the underlying pods are actually ready to serve traffic.
func (r *VirtualMCPServerReconciler) determineStatusFromPods(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	ready, pending, failed int,
) statusDecision {
	// Handle non-ready states first (early returns reduce nesting)
	if ready == 0 {
		if failed > 0 {
			return statusDecision{
				phase:          mcpv1alpha1.VirtualMCPServerPhaseFailed,
				message:        "Virtual MCP server failed to start",
				reason:         "DeploymentFailed",
				conditionMsg:   "Deployment failed",
				conditionState: metav1.ConditionFalse,
			}
		}
		// pending > 0 or no pods at all
		msg := "Virtual MCP server is starting"
		if pending == 0 {
			msg = "No pods found for Virtual MCP server"
		}
		return statusDecision{
			phase:          mcpv1alpha1.VirtualMCPServerPhasePending,
			message:        msg,
			reason:         "DeploymentNotReady",
			conditionMsg:   "Deployment is not yet ready",
			conditionState: metav1.ConditionFalse,
		}
	}

	// Pods are ready (passed readiness probes) - check backend health if backends exist
	if len(vmcp.Status.DiscoveredBackends) == 0 {
		// No backends discovered yet - pods ready is sufficient for Ready
		return statusDecision{
			phase:          mcpv1alpha1.VirtualMCPServerPhaseReady,
			message:        "Virtual MCP server is running",
			reason:         "DeploymentReady",
			conditionMsg:   "Deployment is ready",
			conditionState: metav1.ConditionTrue,
		}
	}

	// Backends exist - determine health status
	return r.determineStatusFromBackends(ctx, vmcp)
}

func (r *VirtualMCPServerReconciler) updateVirtualMCPServerStatus(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	// List the pods for this VirtualMCPServer's deployment
	podList := &corev1.PodList{}
	listOpts := []client.ListOption{
		client.InNamespace(vmcp.Namespace),
		client.MatchingLabels(labelsForVirtualMCPServer(vmcp.Name)),
	}
	if err := r.List(ctx, podList, listOpts...); err != nil {
		return err
	}

	// Count pod states based on actual readiness, not just phase.
	// A pod in Running phase may not be ready to serve traffic if it hasn't
	// passed its readiness probe yet. We must check the PodReady condition.
	var ready, pending, failed int
	for _, pod := range podList.Items {
		// Check for terminal failure states first
		if pod.Status.Phase == corev1.PodFailed {
			failed++
			continue
		}

		// Check if pod is actually ready to serve traffic (passed readiness probes)
		// This is the authoritative signal that the pod can handle requests
		isPodReady := false
		for _, condition := range pod.Status.Conditions {
			if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
				isPodReady = true
				break
			}
		}

		if isPodReady {
			ready++
		} else {
			// Pod exists but isn't ready yet (still starting, or readiness probe failing)
			pending++
		}
	}

	// Determine status in one place (no branching/repetition)
	decision := r.determineStatusFromPods(ctx, vmcp, ready, pending, failed)

	// Apply all status updates at once
	statusManager.SetPhase(decision.phase)
	statusManager.SetMessage(decision.message)
	statusManager.SetReadyCondition(decision.reason, decision.conditionMsg, decision.conditionState)
	statusManager.SetObservedGeneration(vmcp.Generation)

	return nil
}

// labelsForVirtualMCPServer returns the labels for selecting the resources belonging to the given VirtualMCPServer CR name
func labelsForVirtualMCPServer(name string) map[string]string {
	return map[string]string{
		"app":                        "virtualmcpserver",
		"app.kubernetes.io/name":     "virtualmcpserver",
		"app.kubernetes.io/instance": name,
		"toolhive":                   "true",
		"toolhive-name":              name,
	}
}

// vmcpServiceAccountName returns the service account name for the vmcp server
// Uses "-vmcp" suffix to avoid conflicts with MCPServer or MCPRemoteProxy resources of the same name.
// This allows VirtualMCPServer, MCPServer, and MCPRemoteProxy to coexist in the same namespace
// with the same base name (e.g., "foo-vmcp", "foo-proxy-runner", "foo-remote-proxy-runner").
func vmcpServiceAccountName(vmcpName string) string {
	return fmt.Sprintf("%s-vmcp", vmcpName)
}

// outgoingAuthSource returns the outgoing auth source mode with default fallback.
// Returns OutgoingAuthSourceDiscovered if not specified.
func outgoingAuthSource(vmcp *mcpv1alpha1.VirtualMCPServer) string {
	if vmcp.Spec.OutgoingAuth != nil && vmcp.Spec.OutgoingAuth.Source != "" {
		return vmcp.Spec.OutgoingAuth.Source
	}
	return OutgoingAuthSourceDiscovered
}

// serviceAccountNameForVmcp returns the service account name for a VirtualMCPServer.
// - User-provided service account: Returns the user-specified service account name
// - All other modes: Returns the dedicated service account name (for status reporting)
func (*VirtualMCPServerReconciler) serviceAccountNameForVmcp(vmcp *mcpv1alpha1.VirtualMCPServer) string {
	// If a service account is specified, use it
	if vmcp.Spec.ServiceAccount != nil {
		return *vmcp.Spec.ServiceAccount
	}

	// Use dedicated service account with K8s API permissions for status reporting
	// (required in all modes - discovered and inline)
	return vmcpServiceAccountName(vmcp.Name)
}

// vmcpServiceName generates the service name for a VirtualMCPServer
// Uses "vmcp-" prefix to distinguish from MCPServer's "mcp-{name}-proxy" pattern.
// This allows VirtualMCPServer and MCPServer to coexist with the same base name.
//
// Design Note: Each controller has its own service naming functions rather than using a shared utility
// because naming conventions are intentionally different to prevent conflicts:
// - MCPServer: "mcp-{name}-proxy"
// - MCPRemoteProxy: "mcp-{name}-remote-proxy"
// - VirtualMCPServer: "vmcp-{name}"
//
// This pattern is controller-specific by design. Moving to controllerutil would not add value since
// there's no shared logic - just different prefixes/suffixes for each resource type.
func vmcpServiceName(vmcpName string) string {
	return fmt.Sprintf("vmcp-%s", vmcpName)
}

// vmcpConfigMapName generates the ConfigMap name for a VirtualMCPServer's vmcp configuration
// Uses "-vmcp-config" suffix pattern.
func vmcpConfigMapName(vmcpName string) string {
	return fmt.Sprintf("%s-vmcp-config", vmcpName)
}

// createVmcpServiceURL generates the full cluster-local service URL for a VirtualMCPServer
// While the URL pattern (http://{service}.{namespace}.svc.cluster.local:{port}) is standard,
// each controller has different service naming requirements (see vmcpServiceName comment).
func createVmcpServiceURL(vmcpName, namespace string, port int32) string {
	serviceName := vmcpServiceName(vmcpName)
	return fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", serviceName, namespace, port)
}

// convertExternalAuthConfigToStrategy converts an MCPExternalAuthConfig to a BackendAuthStrategy.
// This uses the converter registry to support all auth types (token exchange, header injection, etc.).
// For ConfigMap mode (inline), secrets are referenced as environment variables that will be
// mounted in the deployment. Each ExternalAuthConfig gets a unique env var name to avoid conflicts.
func (*VirtualMCPServerReconciler) convertExternalAuthConfigToStrategy(
	externalAuthConfig *mcpv1alpha1.MCPExternalAuthConfig,
) (*authtypes.BackendAuthStrategy, error) {
	// Use the converter registry to convert to typed strategy
	registry := converters.DefaultRegistry()
	converter, err := registry.GetConverter(externalAuthConfig.Spec.Type)
	if err != nil {
		return nil, err
	}

	// Convert to typed BackendAuthStrategy (this will use env var references for secrets)
	strategy, err := converter.ConvertToStrategy(externalAuthConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to convert external auth config to strategy: %w", err)
	}

	// Set unique env var names per ExternalAuthConfig to avoid conflicts
	// when multiple configs of the same type reference different secrets
	if strategy.TokenExchange != nil &&
		externalAuthConfig.Spec.TokenExchange != nil &&
		externalAuthConfig.Spec.TokenExchange.ClientSecretRef != nil {
		strategy.TokenExchange.ClientSecretEnv = ctrlutil.GenerateUniqueTokenExchangeEnvVarName(externalAuthConfig.Name)
	}
	if strategy.HeaderInjection != nil &&
		externalAuthConfig.Spec.HeaderInjection != nil &&
		externalAuthConfig.Spec.HeaderInjection.ValueSecretRef != nil {
		strategy.HeaderInjection.HeaderValueEnv = ctrlutil.GenerateUniqueHeaderInjectionEnvVarName(externalAuthConfig.Name)
	}

	return strategy, nil
}

// convertBackendAuthConfigToVMCP converts a BackendAuthConfig from CRD to vmcp config.
func (r *VirtualMCPServerReconciler) convertBackendAuthConfigToVMCP(
	ctx context.Context,
	namespace string,
	crdConfig *mcpv1alpha1.BackendAuthConfig,
) (*authtypes.BackendAuthStrategy, error) {
	// For type="discovered", return a minimal strategy (will be populated by discovery)
	if crdConfig.Type == mcpv1alpha1.BackendAuthTypeDiscovered {
		return &authtypes.BackendAuthStrategy{
			Type: crdConfig.Type,
		}, nil
	}

	// For type="external_auth_config_ref", fetch and convert the referenced config
	if crdConfig.ExternalAuthConfigRef != nil {
		// Fetch the MCPExternalAuthConfig and convert it
		externalAuthConfig, err := ctrlutil.GetExternalAuthConfigByName(
			ctx, r.Client, namespace, crdConfig.ExternalAuthConfigRef.Name)
		if err != nil {
			return nil, fmt.Errorf("failed to get MCPExternalAuthConfig %s: %w", crdConfig.ExternalAuthConfigRef.Name, err)
		}

		// Convert the external auth config to strategy
		return r.convertExternalAuthConfigToStrategy(externalAuthConfig)
	}

	// Fallback: return minimal strategy
	return &authtypes.BackendAuthStrategy{
		Type: crdConfig.Type,
	}, nil
}

// listMCPServersAsMap lists all MCPServers in the namespace and returns a map by name.
func (r *VirtualMCPServerReconciler) listMCPServersAsMap(
	ctx context.Context,
	namespace string,
) (map[string]*mcpv1alpha1.MCPServer, error) {
	mcpServerList := &mcpv1alpha1.MCPServerList{}
	if err := r.List(ctx, mcpServerList, client.InNamespace(namespace)); err != nil {
		return nil, err
	}
	mcpServerMap := make(map[string]*mcpv1alpha1.MCPServer, len(mcpServerList.Items))
	for i := range mcpServerList.Items {
		mcpServerMap[mcpServerList.Items[i].Name] = &mcpServerList.Items[i]
	}
	return mcpServerMap, nil
}

// listMCPRemoteProxiesAsMap lists all MCPRemoteProxies in the namespace and returns a map by name.
func (r *VirtualMCPServerReconciler) listMCPRemoteProxiesAsMap(
	ctx context.Context,
	namespace string,
) (map[string]*mcpv1alpha1.MCPRemoteProxy, error) {
	mcpRemoteProxyList := &mcpv1alpha1.MCPRemoteProxyList{}
	if err := r.List(ctx, mcpRemoteProxyList, client.InNamespace(namespace)); err != nil {
		return nil, err
	}
	mcpRemoteProxyMap := make(map[string]*mcpv1alpha1.MCPRemoteProxy, len(mcpRemoteProxyList.Items))
	for i := range mcpRemoteProxyList.Items {
		mcpRemoteProxyMap[mcpRemoteProxyList.Items[i].Name] = &mcpRemoteProxyList.Items[i]
	}
	return mcpRemoteProxyMap, nil
}

// discoverExternalAuthConfigs discovers ExternalAuthConfig from workloads and adds them to the outgoing config.
// Returns a list of non-fatal errors that should be reported via status conditions.
// The controller should continue in degraded mode even if some auth configs fail.
func (r *VirtualMCPServerReconciler) discoverExternalAuthConfigs(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	typedWorkloads []workloads.TypedWorkload,
	outgoing *vmcpconfig.OutgoingAuthConfig,
) ([]string, []AuthConfigError) {
	ctxLogger := log.FromContext(ctx)
	var authErrors []AuthConfigError
	var backendsWithAuthConfig []string

	mcpServerMap, err := r.listMCPServersAsMap(ctx, vmcp.Namespace)
	if err != nil {
		ctxLogger.Error(err, "Failed to list MCPServers")
		return backendsWithAuthConfig, authErrors
	}

	mcpRemoteProxyMap, err := r.listMCPRemoteProxiesAsMap(ctx, vmcp.Namespace)
	if err != nil {
		ctxLogger.Error(err, "Failed to list MCPRemoteProxies")
		return backendsWithAuthConfig, authErrors
	}

	for _, workloadInfo := range typedWorkloads {
		externalAuthConfigName := r.getExternalAuthConfigNameFromWorkload(
			workloadInfo, mcpServerMap, mcpRemoteProxyMap)
		if externalAuthConfigName == "" {
			continue
		}

		// Track that this backend has an auth config (will attempt discovery)
		backendsWithAuthConfig = append(backendsWithAuthConfig, workloadInfo.Name)

		// Fetch the MCPExternalAuthConfig
		externalAuthConfig, err := ctrlutil.GetExternalAuthConfigByName(
			ctx, r.Client, vmcp.Namespace, externalAuthConfigName)
		if err != nil {
			ctxLogger.V(1).Info("Failed to get MCPExternalAuthConfig for backend",
				"backend", workloadInfo.Name,
				"externalAuthConfig", externalAuthConfigName,
				"error", err)
			authErrors = append(authErrors, AuthConfigError{
				Context:     fmt.Sprintf("%s%s", authContextDiscoveredPrefix, workloadInfo.Name),
				BackendName: workloadInfo.Name,
				Error:       fmt.Errorf("failed to get MCPExternalAuthConfig %s: %w", externalAuthConfigName, err),
			})
			continue
		}

		// Convert MCPExternalAuthConfig to BackendAuthStrategy
		strategy, err := r.convertExternalAuthConfigToStrategy(externalAuthConfig)
		if err != nil {
			ctxLogger.V(1).Info("Failed to convert MCPExternalAuthConfig to strategy",
				"backend", workloadInfo.Name,
				"externalAuthConfig", externalAuthConfig.Name,
				"error", err)
			authErrors = append(authErrors, AuthConfigError{
				Context:     fmt.Sprintf("%s%s", authContextDiscoveredPrefix, workloadInfo.Name),
				BackendName: workloadInfo.Name,
				Error:       fmt.Errorf("failed to convert MCPExternalAuthConfig: %w", err),
			})
			continue
		}

		// Only add if not already overridden in inline config
		if vmcp.Spec.OutgoingAuth == nil || vmcp.Spec.OutgoingAuth.Backends == nil {
			outgoing.Backends[workloadInfo.Name] = strategy
		} else if _, exists := vmcp.Spec.OutgoingAuth.Backends[workloadInfo.Name]; !exists {
			// Only add discovered config if not explicitly overridden
			outgoing.Backends[workloadInfo.Name] = strategy
		}
	}

	return backendsWithAuthConfig, authErrors
}

// getExternalAuthConfigNameFromWorkload extracts the ExternalAuthConfigRef name from a workload.
func (*VirtualMCPServerReconciler) getExternalAuthConfigNameFromWorkload(
	workloadInfo workloads.TypedWorkload,
	mcpServerMap map[string]*mcpv1alpha1.MCPServer,
	mcpRemoteProxyMap map[string]*mcpv1alpha1.MCPRemoteProxy,
) string {
	switch workloadInfo.Type {
	case workloads.WorkloadTypeMCPServer:
		mcpServer, found := mcpServerMap[workloadInfo.Name]
		if !found || mcpServer.Spec.ExternalAuthConfigRef == nil {
			return ""
		}
		return mcpServer.Spec.ExternalAuthConfigRef.Name

	case workloads.WorkloadTypeMCPRemoteProxy:
		mcpRemoteProxy, found := mcpRemoteProxyMap[workloadInfo.Name]
		if !found || mcpRemoteProxy.Spec.ExternalAuthConfigRef == nil {
			return ""
		}
		return mcpRemoteProxy.Spec.ExternalAuthConfigRef.Name

	default:
		return ""
	}
}

// buildOutgoingAuthConfig builds an OutgoingAuthConfig from the VirtualMCPServer spec,
// discovering ExternalAuthConfig from MCPServers when source is "discovered".
// Returns the config with partial auth (if some configs fail), backends with auth config,
// and all collected auth errors (non-fatal).
//
// All three types of auth config errors are collected but don't fail reconciliation:
// - Default auth config errors
// - Backend-specific auth config errors (inline overrides)
// - Discovered auth config errors (from ExternalAuthConfigRef)
//
// This allows the system to continue operating in degraded mode with partial auth configuration.
func (r *VirtualMCPServerReconciler) buildOutgoingAuthConfig(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	typedWorkloads []workloads.TypedWorkload,
) (*vmcpconfig.OutgoingAuthConfig, []string, []AuthConfigError) {
	// Determine source - default to "discovered" if not specified
	source := outgoingAuthSource(vmcp)

	outgoing := &vmcpconfig.OutgoingAuthConfig{
		Source:   source,
		Backends: make(map[string]*authtypes.BackendAuthStrategy),
	}

	// Collect all auth config errors (non-fatal)
	var allAuthErrors []AuthConfigError

	// Convert Default if specified
	if vmcp.Spec.OutgoingAuth != nil && vmcp.Spec.OutgoingAuth.Default != nil {
		defaultStrategy, err := r.convertBackendAuthConfigToVMCP(ctx, vmcp.Namespace, vmcp.Spec.OutgoingAuth.Default)
		if err != nil {
			// Collect error but continue (degraded mode)
			allAuthErrors = append(allAuthErrors, AuthConfigError{
				Context:     authContextDefault,
				BackendName: "",
				Error:       fmt.Errorf("failed to convert default auth config: %w", err),
			})
		} else {
			outgoing.Default = defaultStrategy
		}
	}

	// Discover ExternalAuthConfig from MCPServers to populate backend auth configs.
	// This function is called from processOutgoingAuth for both inline and discovered modes:
	// - Inline/static mode: Full backend auth details are embedded in the ConfigMap
	// - Discovered/dynamic mode: Auth configs are validated and errors reported via conditions
	//
	// Discovered errors are collected but don't fail reconciliation (degraded mode).
	backendsWithAuthConfig, discoveredErrors := r.discoverExternalAuthConfigs(ctx, vmcp, typedWorkloads, outgoing)
	allAuthErrors = append(allAuthErrors, discoveredErrors...)

	// Apply inline overrides (works for all source modes)
	if vmcp.Spec.OutgoingAuth != nil && vmcp.Spec.OutgoingAuth.Backends != nil {
		for backendName, backendAuth := range vmcp.Spec.OutgoingAuth.Backends {
			strategy, err := r.convertBackendAuthConfigToVMCP(ctx, vmcp.Namespace, &backendAuth)
			if err != nil {
				// Collect error but continue (degraded mode)
				allAuthErrors = append(allAuthErrors, AuthConfigError{
					Context:     fmt.Sprintf("%s%s", authContextBackendPrefix, backendName),
					BackendName: backendName,
					Error:       fmt.Errorf("failed to convert backend auth config: %w", err),
				})
			} else {
				outgoing.Backends[backendName] = strategy
			}
		}
	}

	return outgoing, backendsWithAuthConfig, allAuthErrors
}

// convertBackendsToStaticBackends converts Backend objects to StaticBackendConfig for ConfigMap embedding.
// Preserves metadata and uses transport types from workload Specs.
// Logs warnings when backends are skipped due to missing URL or transport information.
func convertBackendsToStaticBackends(
	ctx context.Context,
	backends []vmcptypes.Backend,
	transportMap map[string]string,
) []vmcpconfig.StaticBackendConfig {
	logger := log.FromContext(ctx)
	static := make([]vmcpconfig.StaticBackendConfig, 0, len(backends))
	for _, backend := range backends {
		if backend.BaseURL == "" {
			logger.V(1).Info("Skipping backend without URL in static mode",
				"backend", backend.Name)
			continue
		}

		transport := transportMap[backend.Name]
		if transport == "" {
			logger.V(1).Info("Skipping backend without transport information in static mode",
				"backend", backend.Name)
			continue
		}

		static = append(static, vmcpconfig.StaticBackendConfig{
			Name:      backend.Name,
			URL:       backend.BaseURL,
			Transport: transport,
			Metadata:  backend.Metadata,
		})
	}
	return static
}

// validateEmbeddingServerRef validates that the referenced EmbeddingServer exists.
// Readiness gating is handled by isEmbeddingServerReady (called from ensureAllResources),
// ensuring consistent retry behavior (fixed-interval requeue instead of exponential backoff).
func (r *VirtualMCPServerReconciler) validateEmbeddingServerRef(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	ctxLogger := log.FromContext(ctx)

	if vmcp.Spec.EmbeddingServerRef == nil {
		return nil
	}

	refName := vmcp.Spec.EmbeddingServerRef.Name
	es := &mcpv1alpha1.EmbeddingServer{}
	err := r.Get(ctx, types.NamespacedName{
		Name:      refName,
		Namespace: vmcp.Namespace,
	}, es)

	if errors.IsNotFound(err) {
		message := fmt.Sprintf("Referenced EmbeddingServer %s not found", refName)
		statusManager.SetPhase(mcpv1alpha1.VirtualMCPServerPhaseFailed)
		statusManager.SetMessage(message)
		statusManager.SetEmbeddingServerReadyCondition(
			mcpv1alpha1.ConditionReasonEmbeddingServerNotFound,
			message,
			metav1.ConditionFalse,
		)
		statusManager.SetObservedGeneration(vmcp.Generation)
		if r.Recorder != nil {
			r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "EmbeddingServerRefNotFound", "ValidateEmbeddingServerRef",
				"Referenced EmbeddingServer %s not found", refName)
		}
		return err
	} else if err != nil {
		ctxLogger.Error(err, "Failed to get referenced EmbeddingServer", "name", refName)
		return err
	}

	// Existence validated — readiness is checked later by isEmbeddingServerReady
	return nil
}

// mapEmbeddingServerToVirtualMCPServer maps EmbeddingServer changes to VirtualMCPServer
// reconciliation requests. This triggers reconciliation when a referenced EmbeddingServer's
// status changes (e.g., becomes ready or fails).
func (r *VirtualMCPServerReconciler) mapEmbeddingServerToVirtualMCPServer(
	ctx context.Context,
	obj client.Object,
) []reconcile.Request {
	es, ok := obj.(*mcpv1alpha1.EmbeddingServer)
	if !ok {
		return nil
	}

	vmcpList := &mcpv1alpha1.VirtualMCPServerList{}
	if err := r.List(ctx, vmcpList, client.InNamespace(es.Namespace)); err != nil {
		log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for EmbeddingServer watch")
		return nil
	}

	var requests []reconcile.Request
	for _, vmcp := range vmcpList.Items {
		// Only match VirtualMCPServers that reference this EmbeddingServer by name
		if vmcp.Spec.EmbeddingServerRef != nil && vmcp.Spec.EmbeddingServerRef.Name == es.Name {
			requests = append(requests, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      vmcp.Name,
					Namespace: vmcp.Namespace,
				},
			})
		}
	}

	return requests
}

// SetupWithManager sets up the controller with the Manager
func (r *VirtualMCPServerReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&mcpv1alpha1.VirtualMCPServer{}).
		Owns(&appsv1.Deployment{}).
		Owns(&corev1.Service{}).
		Owns(&corev1.ConfigMap{}).
		Watches(&mcpv1alpha1.MCPGroup{}, handler.EnqueueRequestsFromMapFunc(r.mapMCPGroupToVirtualMCPServer)).
		Watches(&mcpv1alpha1.MCPServer{}, handler.EnqueueRequestsFromMapFunc(r.mapMCPServerToVirtualMCPServer)).
		Watches(&mcpv1alpha1.MCPRemoteProxy{}, handler.EnqueueRequestsFromMapFunc(r.mapMCPRemoteProxyToVirtualMCPServer)).
		Watches(&mcpv1alpha1.MCPExternalAuthConfig{}, handler.EnqueueRequestsFromMapFunc(r.mapExternalAuthConfigToVirtualMCPServer)).
		Watches(&mcpv1alpha1.MCPToolConfig{}, handler.EnqueueRequestsFromMapFunc(r.mapToolConfigToVirtualMCPServer)).
		Watches(
			&mcpv1alpha1.VirtualMCPCompositeToolDefinition{},
			handler.EnqueueRequestsFromMapFunc(r.mapCompositeToolDefinitionToVirtualMCPServer),
		).
		// Watch referenced EmbeddingServers so that readiness/status changes
		// trigger VirtualMCPServer reconciliation.
		Watches(
			&mcpv1alpha1.EmbeddingServer{},
			handler.EnqueueRequestsFromMapFunc(r.mapEmbeddingServerToVirtualMCPServer),
		).
		// Watch referenced MCPOIDCConfigs so that validity/hash changes
		// trigger VirtualMCPServer reconciliation.
		Watches(
			&mcpv1alpha1.MCPOIDCConfig{},
			handler.EnqueueRequestsFromMapFunc(r.mapOIDCConfigToVirtualMCPServer),
		).
		Complete(r)
}

// mapMCPGroupToVirtualMCPServer maps MCPGroup changes to VirtualMCPServer reconciliation requests
func (r *VirtualMCPServerReconciler) mapMCPGroupToVirtualMCPServer(ctx context.Context, obj client.Object) []reconcile.Request {
	mcpGroup, ok := obj.(*mcpv1alpha1.MCPGroup)
	if !ok {
		return nil
	}

	vmcpList := &mcpv1alpha1.VirtualMCPServerList{}
	if err := r.List(ctx, vmcpList, client.InNamespace(mcpGroup.Namespace)); err != nil {
		log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for MCPGroup watch")
		return nil
	}

	var requests []reconcile.Request
	for _, vmcp := range vmcpList.Items {
		if vmcp.Spec.Config.Group == mcpGroup.Name {
			requests = append(requests, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      vmcp.Name,
					Namespace: vmcp.Namespace,
				},
			})
		}
	}

	return requests
}

// mapMCPServerToVirtualMCPServer maps MCPServer changes to VirtualMCPServer reconciliation requests.
// This function implements an optimization to only reconcile VirtualMCPServers that are actually
// affected by the MCPServer change, rather than reconciling all VirtualMCPServers in the namespace.
//
// The optimization works by:
// 1. Finding all MCPGroups that include the changed MCPServer (via Status.Servers)
// 2. Finding all VirtualMCPServers that reference those MCPGroups
// 3. Only reconciling those specific VirtualMCPServers
//
// This significantly reduces unnecessary reconciliations in large clusters with many VirtualMCPServers.
func (r *VirtualMCPServerReconciler) mapMCPServerToVirtualMCPServer(ctx context.Context, obj client.Object) []reconcile.Request {
	mcpServer, ok := obj.(*mcpv1alpha1.MCPServer)
	if !ok {
		return nil
	}

	ctxLogger := log.FromContext(ctx)

	// Step 1: Find all MCPGroups that include this MCPServer
	// MCPGroups track their member servers in Status.Servers (populated by MCPGroup controller)
	mcpGroupList := &mcpv1alpha1.MCPGroupList{}
	if err := r.List(ctx, mcpGroupList, client.InNamespace(mcpServer.Namespace)); err != nil {
		ctxLogger.Error(err, "Failed to list MCPGroups for MCPServer watch")
		return nil
	}

	// Track which MCPGroups include this MCPServer
	affectedGroups := make(map[string]bool)
	for _, group := range mcpGroupList.Items {
		// Check if this MCPServer is in the group's server list
		for _, serverName := range group.Status.Servers {
			if serverName == mcpServer.Name {
				affectedGroups[group.Name] = true
				ctxLogger.V(1).Info("MCPServer is member of MCPGroup",
					"mcpServer", mcpServer.Name,
					"mcpGroup", group.Name)
				break // No need to check other servers in this group
			}
		}
	}

	// If no groups include this MCPServer, no VirtualMCPServers need reconciliation
	if len(affectedGroups) == 0 {
		ctxLogger.V(1).Info("MCPServer not a member of any MCPGroup, skipping VirtualMCPServer reconciliation",
			"mcpServer", mcpServer.Name)
		return nil
	}

	// Step 2: Find VirtualMCPServers that reference the affected MCPGroups
	vmcpList := &mcpv1alpha1.VirtualMCPServerList{}
	if err := r.List(ctx, vmcpList, client.InNamespace(mcpServer.Namespace)); err != nil {
		ctxLogger.Error(err, "Failed to list VirtualMCPServers for MCPServer watch")
		return nil
	}

	var requests []reconcile.Request
	for _, vmcp := range vmcpList.Items {
		// Only reconcile if this VirtualMCPServer references an affected MCPGroup
		if affectedGroups[vmcp.Spec.Config.Group] {
			requests = append(requests, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      vmcp.Name,
					Namespace: vmcp.Namespace,
				},
			})
			ctxLogger.V(1).Info("Queuing VirtualMCPServer for reconciliation due to MCPServer change",
				"virtualMCPServer", vmcp.Name,
				"mcpGroup", vmcp.Spec.Config.Group,
				"mcpServer", mcpServer.Name)
		}
	}

	ctxLogger.V(1).Info("Mapped MCPServer to VirtualMCPServers",
		"mcpServer", mcpServer.Name,
		"affectedGroups", len(affectedGroups),
		"virtualMCPServers", len(requests))

	return requests
}

// mapMCPRemoteProxyToVirtualMCPServer maps MCPRemoteProxy changes to VirtualMCPServer reconciliation requests.
// This function implements the same optimization as mapMCPServerToVirtualMCPServer to only reconcile
// VirtualMCPServers that are actually affected by the MCPRemoteProxy change.
//
// The optimization works by:
// 1. Finding all MCPGroups that include the changed MCPRemoteProxy (via Status.RemoteProxies)
// 2. Finding all VirtualMCPServers that reference those MCPGroups
// 3. Only reconciling those specific VirtualMCPServers
func (r *VirtualMCPServerReconciler) mapMCPRemoteProxyToVirtualMCPServer(
	ctx context.Context,
	obj client.Object,
) []reconcile.Request {
	mcpRemoteProxy, ok := obj.(*mcpv1alpha1.MCPRemoteProxy)
	if !ok {
		return nil
	}

	ctxLogger := log.FromContext(ctx)

	// Step 1: Find all MCPGroups that include this MCPRemoteProxy
	// MCPGroups track their member remote proxies in Status.RemoteProxies (populated by MCPGroup controller)
	mcpGroupList := &mcpv1alpha1.MCPGroupList{}
	if err := r.List(ctx, mcpGroupList, client.InNamespace(mcpRemoteProxy.Namespace)); err != nil {
		ctxLogger.Error(err, "Failed to list MCPGroups for MCPRemoteProxy watch")
		return nil
	}

	// Track which MCPGroups include this MCPRemoteProxy
	affectedGroups := make(map[string]bool)
	for _, group := range mcpGroupList.Items {
		// Check if this MCPRemoteProxy is in the group's remote proxy list
		for _, proxyName := range group.Status.RemoteProxies {
			if proxyName == mcpRemoteProxy.Name {
				affectedGroups[group.Name] = true
				ctxLogger.V(1).Info("MCPRemoteProxy is member of MCPGroup",
					"mcpRemoteProxy", mcpRemoteProxy.Name,
					"mcpGroup", group.Name)
				break // No need to check other proxies in this group
			}
		}
	}

	// If no groups include this MCPRemoteProxy, no VirtualMCPServers need reconciliation
	if len(affectedGroups) == 0 {
		ctxLogger.V(1).Info("MCPRemoteProxy not a member of any MCPGroup, skipping VirtualMCPServer reconciliation",
			"mcpRemoteProxy", mcpRemoteProxy.Name)
		return nil
	}

	// Step 2: Find VirtualMCPServers that reference the affected MCPGroups
	vmcpList := &mcpv1alpha1.VirtualMCPServerList{}
	if err := r.List(ctx, vmcpList, client.InNamespace(mcpRemoteProxy.Namespace)); err != nil {
		ctxLogger.Error(err, "Failed to list VirtualMCPServers for MCPRemoteProxy watch")
		return nil
	}

	var requests []reconcile.Request
	for _, vmcp := range vmcpList.Items {
		// Only reconcile if this VirtualMCPServer references an affected MCPGroup
		if affectedGroups[vmcp.Spec.Config.Group] {
			requests = append(requests, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      vmcp.Name,
					Namespace: vmcp.Namespace,
				},
			})
			ctxLogger.V(1).Info("Queuing VirtualMCPServer for reconciliation due to MCPRemoteProxy change",
				"virtualMCPServer", vmcp.Name,
				"mcpGroup", vmcp.Spec.Config.Group,
				"mcpRemoteProxy", mcpRemoteProxy.Name)
		}
	}

	ctxLogger.V(1).Info("Mapped MCPRemoteProxy to VirtualMCPServers",
		"mcpRemoteProxy", mcpRemoteProxy.Name,
		"affectedGroups", len(affectedGroups),
		"virtualMCPServers", len(requests))

	return requests
}

// mapExternalAuthConfigToVirtualMCPServer maps MCPExternalAuthConfig changes to VirtualMCPServer reconciliation requests
func (r *VirtualMCPServerReconciler) mapExternalAuthConfigToVirtualMCPServer(
	ctx context.Context,
	obj client.Object,
) []reconcile.Request {
	externalAuthConfig, ok := obj.(*mcpv1alpha1.MCPExternalAuthConfig)
	if !ok {
		return nil
	}

	vmcpList := &mcpv1alpha1.VirtualMCPServerList{}
	if err := r.List(ctx, vmcpList, client.InNamespace(externalAuthConfig.Namespace)); err != nil {
		log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for MCPExternalAuthConfig watch")
		return nil
	}

	var requests []reconcile.Request
	for _, vmcp := range vmcpList.Items {
		// Only reconcile VirtualMCPServers that actually reference this ExternalAuthConfig
		// This includes both inline references and discovered references (via MCPServers)
		if r.vmcpReferencesExternalAuthConfig(ctx, &vmcp, externalAuthConfig.Name) {
			requests = append(requests, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      vmcp.Name,
					Namespace: vmcp.Namespace,
				},
			})
		}
	}

	return requests
}

// mapToolConfigToVirtualMCPServer maps MCPToolConfig changes to VirtualMCPServer reconciliation requests
func (r *VirtualMCPServerReconciler) mapToolConfigToVirtualMCPServer(ctx context.Context, obj client.Object) []reconcile.Request {
	toolConfig, ok := obj.(*mcpv1alpha1.MCPToolConfig)
	if !ok {
		return nil
	}

	vmcpList := &mcpv1alpha1.VirtualMCPServerList{}
	if err := r.List(ctx, vmcpList, client.InNamespace(toolConfig.Namespace)); err != nil {
		log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for MCPToolConfig watch")
		return nil
	}

	var requests []reconcile.Request
	for _, vmcp := range vmcpList.Items {
		if r.vmcpReferencesToolConfig(&vmcp, toolConfig.Name) {
			requests = append(requests, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      vmcp.Name,
					Namespace: vmcp.Namespace,
				},
			})
		}
	}

	return requests
}

// vmcpReferencesToolConfig checks if a VirtualMCPServer references the given MCPToolConfig
func (*VirtualMCPServerReconciler) vmcpReferencesToolConfig(vmcp *mcpv1alpha1.VirtualMCPServer, toolConfigName string) bool {
	if vmcp.Spec.Config.Aggregation == nil || len(vmcp.Spec.Config.Aggregation.Tools) == 0 {
		return false
	}

	for _, tc := range vmcp.Spec.Config.Aggregation.Tools {
		if tc.ToolConfigRef != nil && tc.ToolConfigRef.Name == toolConfigName {
			return true
		}
	}

	return false
}

// vmcpReferencesExternalAuthConfig checks if a VirtualMCPServer references the given MCPExternalAuthConfig.
// It checks authServerConfigRef, inline references (in outgoingAuth spec), and discovered references
// (via MCPServers in the group).
func (r *VirtualMCPServerReconciler) vmcpReferencesExternalAuthConfig(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	authConfigName string,
) bool {
	// Note: AuthServerConfig is inline (not a ref), so it doesn't reference
	// MCPExternalAuthConfig resources. Only outgoing auth refs are checked here.

	if vmcp.Spec.OutgoingAuth == nil {
		return false
	}

	// Check inline references in outgoing auth configuration
	// Check default backend auth configuration
	if vmcp.Spec.OutgoingAuth.Default != nil &&
		vmcp.Spec.OutgoingAuth.Default.ExternalAuthConfigRef != nil &&
		vmcp.Spec.OutgoingAuth.Default.ExternalAuthConfigRef.Name == authConfigName {
		return true
	}

	// Check per-backend auth configurations
	for _, backendAuth := range vmcp.Spec.OutgoingAuth.Backends {
		if backendAuth.ExternalAuthConfigRef != nil &&
			backendAuth.ExternalAuthConfigRef.Name == authConfigName {
			return true
		}
	}

	// Check discovered references when source is "discovered"
	// When using discovered mode, auth configs are referenced through MCPServers, not inline
	if vmcp.Spec.OutgoingAuth.Source == OutgoingAuthSourceDiscovered {
		if r.mcpGroupBackendsReferenceExternalAuthConfig(ctx, vmcp, authConfigName) {
			return true
		}
	}

	return false
}

// mcpGroupBackendsReferenceExternalAuthConfig checks if any MCPServers or MCPRemoteProxies
// in the VirtualMCPServer's group reference the given MCPExternalAuthConfig
func (r *VirtualMCPServerReconciler) mcpGroupBackendsReferenceExternalAuthConfig(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	authConfigName string,
) bool {
	ctxLogger := log.FromContext(ctx)

	// Get the MCPGroup to verify it exists
	mcpGroup := &mcpv1alpha1.MCPGroup{}
	err := r.Get(ctx, types.NamespacedName{
		Name:      vmcp.Spec.Config.Group,
		Namespace: vmcp.Namespace,
	}, mcpGroup)
	if err != nil {
		// If we can't get the group, we can't determine if it references the auth config
		// Return false to avoid false positives
		ctxLogger.Error(err, "Failed to get MCPGroup for ExternalAuthConfig reference check",
			"group", vmcp.Spec.Config.Group,
			"vmcp", vmcp.Name)
		return false
	}

	listOpts := []client.ListOption{
		client.InNamespace(vmcp.Namespace),
		client.MatchingFields{"spec.groupRef": mcpGroup.Name},
	}

	// List all MCPServers in the group using field selector (same as MCPGroup controller)
	mcpServerList := &mcpv1alpha1.MCPServerList{}
	err = r.List(ctx, mcpServerList, listOpts...)
	if err != nil {
		ctxLogger.Error(err, "Failed to list MCPServers for ExternalAuthConfig reference check",
			"group", mcpGroup.Name)
		return false
	}

	// Check if any MCPServer references the ExternalAuthConfig
	for _, mcpServer := range mcpServerList.Items {
		if mcpServer.Spec.ExternalAuthConfigRef != nil &&
			mcpServer.Spec.ExternalAuthConfigRef.Name == authConfigName {
			return true
		}
	}

	// List all MCPRemoteProxies in the group
	mcpRemoteProxyList := &mcpv1alpha1.MCPRemoteProxyList{}
	err = r.List(ctx, mcpRemoteProxyList, listOpts...)
	if err != nil {
		ctxLogger.Error(err, "Failed to list MCPRemoteProxies for ExternalAuthConfig reference check",
			"group", mcpGroup.Name)
		return false
	}

	// Check if any MCPRemoteProxy references the ExternalAuthConfig
	for _, mcpRemoteProxy := range mcpRemoteProxyList.Items {
		if mcpRemoteProxy.Spec.ExternalAuthConfigRef != nil &&
			mcpRemoteProxy.Spec.ExternalAuthConfigRef.Name == authConfigName {
			return true
		}
	}

	return false
}

// mapCompositeToolDefinitionToVirtualMCPServer maps VirtualMCPCompositeToolDefinition changes to
// VirtualMCPServer reconciliation requests
func (r *VirtualMCPServerReconciler) mapCompositeToolDefinitionToVirtualMCPServer(
	ctx context.Context,
	obj client.Object,
) []reconcile.Request {
	compositeToolDef, ok := obj.(*mcpv1alpha1.VirtualMCPCompositeToolDefinition)
	if !ok {
		return nil
	}

	vmcpList := &mcpv1alpha1.VirtualMCPServerList{}
	if err := r.List(ctx, vmcpList, client.InNamespace(compositeToolDef.Namespace)); err != nil {
		log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for VirtualMCPCompositeToolDefinition watch")
		return nil
	}

	var requests []reconcile.Request
	for _, vmcp := range vmcpList.Items {
		if r.vmcpReferencesCompositeToolDefinition(&vmcp, compositeToolDef.Name) {
			requests = append(requests, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      vmcp.Name,
					Namespace: vmcp.Namespace,
				},
			})
		}
	}

	return requests
}

// vmcpReferencesCompositeToolDefinition checks if a VirtualMCPServer references the given VirtualMCPCompositeToolDefinition
func (*VirtualMCPServerReconciler) vmcpReferencesCompositeToolDefinition(
	vmcp *mcpv1alpha1.VirtualMCPServer,
	compositeToolDefName string,
) bool {
	if len(vmcp.Spec.Config.CompositeToolRefs) == 0 {
		return false
	}

	for i := range vmcp.Spec.Config.CompositeToolRefs {
		if vmcp.Spec.Config.CompositeToolRefs[i].Name == compositeToolDefName {
			return true
		}
	}

	return false
}

// setAuthConfigConditions sets status conditions for all auth config types.
// This ensures conditions reflect the current state by setting:
// - True (ConversionSucceeded) for valid auth configs
// - False (ConversionFailed) for auth config errors
//
// Handles three types of auth config conditions:
// 1. DefaultAuthConfig - for default auth config in OutgoingAuth.Default
// 2. BackendAuthConfig-<name> - for inline backend-specific auth configs in OutgoingAuth.Backends
// 3. DiscoveredAuthConfig-<name> - for discovered auth configs via ExternalAuthConfigRef
//
// This allows users to see the current auth config state for each component via kubectl
// and ensures stale failure conditions are cleared when auth configs are fixed or backends removed.
//
// All auth config errors are non-fatal - the system continues operating in degraded mode.
func setAuthConfigConditions(
	statusManager virtualmcpserverstatus.StatusManager,
	backendsWithAuthConfig []string,
	inlineBackendNames []string,
	hasValidDefaultAuth bool,
	validInlineBackends []string,
	allAuthErrors []AuthConfigError,
) {
	// Build error maps by context for quick lookup
	var defaultAuthError error
	backendAuthErrors := make(map[string]error)
	discoveredAuthErrors := make(map[string]error)

	for _, authError := range allAuthErrors {
		if authError.Context == authContextDefault {
			defaultAuthError = authError.Error
		} else if strings.HasPrefix(authError.Context, authContextBackendPrefix) {
			backendAuthErrors[authError.BackendName] = authError.Error
		} else if strings.HasPrefix(authError.Context, authContextDiscoveredPrefix) {
			discoveredAuthErrors[authError.BackendName] = authError.Error
		}
	}

	// Handle DefaultAuthConfig condition
	if defaultAuthError != nil {
		// Default auth has error - set False condition
		statusManager.SetAuthConfigCondition(
			"DefaultAuthConfig",
			"ConversionFailed",
			fmt.Sprintf("Failed to convert default auth config: %v", defaultAuthError),
			metav1.ConditionFalse,
		)
	} else if hasValidDefaultAuth {
		// Default auth is valid - set True condition
		statusManager.SetAuthConfigCondition(
			"DefaultAuthConfig",
			"ConversionSucceeded",
			"Default auth config is valid",
			metav1.ConditionTrue,
		)
	} else {
		// No default auth configured - remove the condition if it exists
		// This handles cases where:
		// - Auth is completely disabled
		// - Default auth was removed from the spec
		statusManager.RemoveConditionsWithPrefix("DefaultAuthConfig", []string{})
	}

	// Build list of current DiscoveredAuthConfig conditions to preserve
	currentDiscoveredConditions := make([]string, len(backendsWithAuthConfig))
	for i, backendName := range backendsWithAuthConfig {
		currentDiscoveredConditions[i] = fmt.Sprintf("DiscoveredAuthConfig-%s", backendName)
	}

	// Build list of current BackendAuthConfig conditions to preserve
	currentBackendConditions := make([]string, len(inlineBackendNames))
	for i, backendName := range inlineBackendNames {
		currentBackendConditions[i] = fmt.Sprintf("BackendAuthConfig-%s", backendName)
	}

	// Remove stale conditions for backends that no longer exist in the spec
	statusManager.RemoveConditionsWithPrefix("DiscoveredAuthConfig-", currentDiscoveredConditions)
	statusManager.RemoveConditionsWithPrefix("BackendAuthConfig-", currentBackendConditions)

	// Set DiscoveredAuthConfig conditions for backends with ExternalAuthConfigRef
	for _, backendName := range backendsWithAuthConfig {
		conditionType := fmt.Sprintf("DiscoveredAuthConfig-%s", backendName)

		if err, hasError := discoveredAuthErrors[backendName]; hasError {
			// Backend has discovered auth config error - set False condition
			statusManager.SetAuthConfigCondition(
				conditionType,
				"ConversionFailed",
				fmt.Sprintf("Failed to convert discovered auth config: %v", err),
				metav1.ConditionFalse,
			)
		} else {
			// Backend has valid discovered auth config - set True condition
			statusManager.SetAuthConfigCondition(
				conditionType,
				"ConversionSucceeded",
				"Discovered auth config is valid",
				metav1.ConditionTrue,
			)
		}
	}

	// Set BackendAuthConfig conditions for inline backend-specific auth configs
	// First, set error conditions
	for backendName, err := range backendAuthErrors {
		conditionType := fmt.Sprintf("BackendAuthConfig-%s", backendName)
		statusManager.SetAuthConfigCondition(
			conditionType,
			"ConversionFailed",
			fmt.Sprintf("Failed to convert backend auth config: %v", err),
			metav1.ConditionFalse,
		)
	}
	// Then, set success conditions for valid backends
	for _, backendName := range validInlineBackends {
		// Skip if this backend has an error (already set above)
		if _, hasError := backendAuthErrors[backendName]; hasError {
			continue
		}
		conditionType := fmt.Sprintf("BackendAuthConfig-%s", backendName)
		statusManager.SetAuthConfigCondition(
			conditionType,
			"ConversionSucceeded",
			"Backend auth config is valid",
			metav1.ConditionTrue,
		)
	}

	// Note: We don't modify the overall AuthConfigured condition here because
	// auth config errors are non-fatal. The system can continue operating with
	// the auth configs that are valid.
}

// generateHMACSecret generates a cryptographically secure 32-byte HMAC secret
// encoded as base64. This secret is used for session token binding in Session Management V2.
//
// Returns a base64-encoded string suitable for use as VMCP_SESSION_HMAC_SECRET.
func generateHMACSecret() (string, error) {
	// Generate 32 bytes of cryptographically secure random data
	secret := make([]byte, 32)
	if _, err := rand.Read(secret); err != nil {
		return "", fmt.Errorf("failed to generate random bytes: %w", err)
	}

	// Encode as base64 for safe storage and environment variable use
	return base64.StdEncoding.EncodeToString(secret), nil
}

// handleOIDCConfig validates and tracks the hash of the referenced MCPOIDCConfig.
// It sets the OIDCConfigRefValidated condition and triggers reconciliation when
// the OIDC configuration changes.
func (r *VirtualMCPServerReconciler) handleOIDCConfig(
	ctx context.Context,
	vmcp *mcpv1alpha1.VirtualMCPServer,
	statusManager virtualmcpserverstatus.StatusManager,
) error {
	ctxLogger := log.FromContext(ctx)

	if vmcp.Spec.IncomingAuth == nil || vmcp.Spec.IncomingAuth.OIDCConfigRef == nil {
		// No MCPOIDCConfig referenced, clear any stored hash
		if vmcp.Status.OIDCConfigHash != "" {
			vmcp.Status.OIDCConfigHash = ""
			if err := r.Status().Update(ctx, vmcp); err != nil {
				return fmt.Errorf("failed to clear MCPOIDCConfig hash from status: %w", err)
			}
		}
		return nil
	}

	ref := vmcp.Spec.IncomingAuth.OIDCConfigRef

	// Get the referenced MCPOIDCConfig
	oidcConfig, err := ctrlutil.GetOIDCConfigForServer(ctx, r.Client, vmcp.Namespace, ref)
	if err != nil {
		statusManager.SetCondition(
			mcpv1alpha1.ConditionOIDCConfigRefValidated,
			mcpv1alpha1.ConditionReasonOIDCConfigRefNotFound,
			fmt.Sprintf("MCPOIDCConfig %s not found: %v", ref.Name, err),
			metav1.ConditionFalse,
		)
		return err
	}

	if oidcConfig == nil {
		statusManager.SetCondition(
			mcpv1alpha1.ConditionOIDCConfigRefValidated,
			mcpv1alpha1.ConditionReasonOIDCConfigRefNotFound,
			fmt.Sprintf("MCPOIDCConfig %s not found", ref.Name),
			metav1.ConditionFalse,
		)
		return fmt.Errorf("MCPOIDCConfig %s not found", ref.Name)
	}

	// Check that the MCPOIDCConfig is valid
	validCondition := meta.FindStatusCondition(oidcConfig.Status.Conditions, mcpv1alpha1.ConditionTypeOIDCConfigReady)
	if validCondition == nil || validCondition.Status != metav1.ConditionTrue {
		msg := fmt.Sprintf("MCPOIDCConfig %s is not valid", ref.Name)
		if validCondition != nil {
			msg = fmt.Sprintf("MCPOIDCConfig %s is not valid: %s", ref.Name, validCondition.Message)
		}
		statusManager.SetCondition(
			mcpv1alpha1.ConditionOIDCConfigRefValidated,
			mcpv1alpha1.ConditionReasonOIDCConfigRefNotValid,
			msg,
			metav1.ConditionFalse,
		)
		return fmt.Errorf("%s", msg)
	}

	// Update ReferencingWorkloads on the MCPOIDCConfig status
	if err := r.updateOIDCConfigReferencingWorkloads(ctx, oidcConfig, vmcp.Name); err != nil {
		ctxLogger.Error(err, "Failed to update MCPOIDCConfig ReferencingWorkloads")
		// Non-fatal: continue with reconciliation
	}

	// Set valid condition
	statusManager.SetCondition(
		mcpv1alpha1.ConditionOIDCConfigRefValidated,
		mcpv1alpha1.ConditionReasonOIDCConfigRefValid,
		fmt.Sprintf("MCPOIDCConfig %s is valid and ready", ref.Name),
		metav1.ConditionTrue,
	)

	// Check if the MCPOIDCConfig hash has changed
	if vmcp.Status.OIDCConfigHash != oidcConfig.Status.ConfigHash {
		ctxLogger.Info("MCPOIDCConfig has changed, updating VirtualMCPServer",
			"vmcp", vmcp.Name,
			"oidcConfig", oidcConfig.Name,
			"oldHash", vmcp.Status.OIDCConfigHash,
			"newHash", oidcConfig.Status.ConfigHash)

		vmcp.Status.OIDCConfigHash = oidcConfig.Status.ConfigHash
		if err := r.Status().Update(ctx, vmcp); err != nil {
			return fmt.Errorf("failed to update MCPOIDCConfig hash in status: %w", err)
		}
	}

	return nil
}

// updateOIDCConfigReferencingWorkloads ensures the VirtualMCPServer is listed in
// the MCPOIDCConfig's ReferencingWorkloads status field.
func (r *VirtualMCPServerReconciler) updateOIDCConfigReferencingWorkloads(
	ctx context.Context,
	oidcConfig *mcpv1alpha1.MCPOIDCConfig,
	vmcpName string,
) error {
	ref := mcpv1alpha1.WorkloadReference{Kind: mcpv1alpha1.WorkloadKindVirtualMCPServer, Name: vmcpName}
	// Check if already listed
	for _, entry := range oidcConfig.Status.ReferencingWorkloads {
		if entry.Kind == ref.Kind && entry.Name == ref.Name {
			return nil
		}
	}

	// Add the workload reference
	oidcConfig.Status.ReferencingWorkloads = append(oidcConfig.Status.ReferencingWorkloads, ref)
	if err := r.Status().Update(ctx, oidcConfig); err != nil {
		return fmt.Errorf("failed to update MCPOIDCConfig ReferencingWorkloads: %w", err)
	}

	return nil
}

// mapOIDCConfigToVirtualMCPServer maps MCPOIDCConfig changes to VirtualMCPServer reconciliation requests.
func (r *VirtualMCPServerReconciler) mapOIDCConfigToVirtualMCPServer(
	ctx context.Context, obj client.Object,
) []reconcile.Request {
	oidcConfig, ok := obj.(*mcpv1alpha1.MCPOIDCConfig)
	if !ok {
		return nil
	}

	vmcpList := &mcpv1alpha1.VirtualMCPServerList{}
	if err := r.List(ctx, vmcpList, client.InNamespace(oidcConfig.Namespace)); err != nil {
		log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for MCPOIDCConfig watch")
		return nil
	}

	var requests []reconcile.Request
	for _, vmcp := range vmcpList.Items {
		if vmcp.Spec.IncomingAuth != nil &&
			vmcp.Spec.IncomingAuth.OIDCConfigRef != nil &&
			vmcp.Spec.IncomingAuth.OIDCConfigRef.Name == oidcConfig.Name {
			requests = append(requests, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      vmcp.Name,
					Namespace: vmcp.Namespace,
				},
			})
		}
	}

	return requests
}