use confimap to track gateway infra

Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com>

Author: zhaohuabing

internal/infrastructure/kubernetes/infra.go

@@ -59,6 +59,10 @@ type Infra struct {
 	// Client wrap k8s client.
 	Client *InfraClient
 
+	// APIReader performs uncached reads/writes for narrow correctness-sensitive
+	// tracking state in the controller namespace. Defaults to the main client in tests.
+	APIReader client.Client
+
 	logger logging.Logger
 
 	// errors is the notifier used to send async errors to the main control loop.
@@ -74,6 +78,7 @@ func NewInfra(cli client.Client, cfg *config.Server, errors message.RunnerErrorN
 		DNSDomain:           cfg.DNSDomain,
 		EnvoyGateway:        cfg.EnvoyGateway,
 		Client:              New(cli),
+		APIReader:           cli,
 		logger:              cfg.Logger.WithName(string(egv1a1.LogComponentInfrastructureRunner)),
 		errors:              errors,
 	}
@@ -85,6 +90,31 @@ func (i *Infra) Close() error { return nil }
 // createOrUpdate creates a ServiceAccount/ConfigMap/Deployment/Service in the kube api server based on the
 // provided ResourceRender, if it doesn't exist and updates it if it does.
 func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error {
+	// Track the last applied proxy resource names in a controller-owned ConfigMap
+	// so renamed resources can be deleted directly by name on the next reconcile.
+	// This avoids relying solely on DeleteAllExcept's cached List view to
+	// discover stale custom-named resources, while keeping that path as a
+	// fallback for older untracked leftovers.
+	var (
+		previousRefs []proxy.ManagedProxyResourceRef
+		currentRefs  []proxy.ManagedProxyResourceRef
+		trackingName string
+		err          error
+	)
+	if proxyRender, ok := r.(*proxy.ResourceRender); ok {
+		trackingName = proxyRender.TrackingConfigMapName()
+		if len(trackingName) > 0 {
+			previousRefs, err = i.loadManagedProxyRefs(ctx, trackingName)
+			if err != nil {
+				return fmt.Errorf("failed to load tracked proxy resource refs for %s: %w", trackingName, err)
+			}
+		}
+		currentRefs, err = proxyRender.ManagedProxyResourceRefs()
+		if err != nil {
+			return fmt.Errorf("failed to compute managed proxy resource refs for %s/%s: %w", proxyRender.Namespace(), proxyRender.Name(), err)
+		}
+	}
+
 	if err := i.createOrUpdateServiceAccount(ctx, r); err != nil {
 		return fmt.Errorf("failed to create or update serviceaccount %s/%s: %w", r.Namespace(), r.Name(), err)
 	}
@@ -113,6 +143,15 @@ func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error {
 		return fmt.Errorf("failed to create or update pdb %s/%s: %w", r.Namespace(), r.Name(), err)
 	}
 
+	if len(trackingName) > 0 {
+		if err := i.deleteTrackedProxyResources(ctx, previousRefs, currentRefs); err != nil {
+			return fmt.Errorf("failed to delete previously tracked proxy resources for %s: %w", trackingName, err)
+		}
+		if err := i.persistManagedProxyRefs(ctx, trackingName, currentRefs); err != nil {
+			return fmt.Errorf("failed to persist tracked proxy resources for %s: %w", trackingName, err)
+		}
+	}
+
 	return nil
 }
 
@@ -146,5 +185,13 @@ func (i *Infra) delete(ctx context.Context, r ResourceRender) error {
 		return fmt.Errorf("failed to delete pdb %s/%s: %w", r.Namespace(), r.Name(), err)
 	}
 
+	if proxyRender, ok := r.(*proxy.ResourceRender); ok {
+		if trackingName := proxyRender.TrackingConfigMapName(); len(trackingName) > 0 {
+			if err := i.deleteTrackedProxyRefState(ctx, trackingName); err != nil {
+				return fmt.Errorf("failed to delete tracked proxy ref state %s: %w", trackingName, err)
+			}
+		}
+	}
+
 	return nil
 }

internal/infrastructure/kubernetes/proxy/resource_provider.go

@@ -7,6 +7,7 @@ package proxy
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"strconv"
 	"time"
@@ -77,6 +78,12 @@ type ResourceRender struct {
 	ownerReferenceUID map[string]types.UID
 }
 
+type ManagedProxyResourceRef struct {
+	Kind      string `json:"kind"`
+	Namespace string `json:"namespace"`
+	Name      string `json:"name"`
+}
+
 // KubernetesInfraProvider provide information for initializing the proxy resource render.
 type KubernetesInfraProvider interface {
 	GetControllerNamespace() string
@@ -135,6 +142,98 @@ func (r *ResourceRender) LabelSelector() labels.Selector {
 	return labels.SelectorFromSet(r.stableSelector().MatchLabels)
 }
 
+func (r *ResourceRender) OwningGatewayNN() *types.NamespacedName {
+	labels := r.infra.GetProxyMetadata().Labels
+	name := labels[gatewayapi.OwningGatewayNameLabel]
+	namespace := labels[gatewayapi.OwningGatewayNamespaceLabel]
+	if len(name) == 0 || len(namespace) == 0 {
+		return nil
+	}
+
+	return &types.NamespacedName{Namespace: namespace, Name: name}
+}
+
+// TrackingOwner returns the logical owner identity used to persist proxy infra
+// tracking state. In normal mode infra is rendered per Gateway even if the
+// effective EnvoyProxy config comes from a GatewayClass, so tracked cleanup must
+// stay isolated per Gateway. Merged gateways intentionally render one shared
+// infra set for the GatewayClass and only carry OwningGatewayClassLabel.
+func (r *ResourceRender) TrackingOwner() (kind string, nn types.NamespacedName, ok bool) {
+	if gwNN := r.OwningGatewayNN(); gwNN != nil {
+		return "Gateway", *gwNN, true
+	}
+
+	gcName := r.infra.GetProxyMetadata().Labels[gatewayapi.OwningGatewayClassLabel]
+	if len(gcName) == 0 {
+		return "", types.NamespacedName{}, false
+	}
+
+	return "GatewayClass", types.NamespacedName{Name: gcName}, true
+}
+
+func (r *ResourceRender) TrackingConfigMapName() string {
+	kind, nn, ok := r.TrackingOwner()
+	if !ok {
+		return ""
+	}
+
+	key := kind + "/" + nn.Namespace + "/" + nn.Name
+	sum := sha256.Sum256([]byte(key))
+	return fmt.Sprintf("envoy-proxy-tracking-%x", sum[:8])
+}
+
+func (r *ResourceRender) ManagedProxyResourceRefs() ([]ManagedProxyResourceRef, error) {
+	refs := make([]ManagedProxyResourceRef, 0, 5)
+
+	sa, err := r.ServiceAccount()
+	if err != nil {
+		return nil, err
+	}
+	refs = append(refs, ManagedProxyResourceRef{Kind: "ServiceAccount", Namespace: sa.Namespace, Name: sa.Name})
+
+	svc, err := r.Service()
+	if err != nil {
+		return nil, err
+	}
+	if svc != nil {
+		refs = append(refs, ManagedProxyResourceRef{Kind: "Service", Namespace: svc.Namespace, Name: svc.Name})
+	}
+
+	deployment, err := r.Deployment()
+	if err != nil {
+		return nil, err
+	}
+	if deployment != nil {
+		refs = append(refs, ManagedProxyResourceRef{Kind: "Deployment", Namespace: deployment.Namespace, Name: deployment.Name})
+	}
+
+	daemonSet, err := r.DaemonSet()
+	if err != nil {
+		return nil, err
+	}
+	if daemonSet != nil {
+		refs = append(refs, ManagedProxyResourceRef{Kind: "DaemonSet", Namespace: daemonSet.Namespace, Name: daemonSet.Name})
+	}
+
+	hpa, err := r.HorizontalPodAutoscaler()
+	if err != nil {
+		return nil, err
+	}
+	if hpa != nil {
+		refs = append(refs, ManagedProxyResourceRef{Kind: "HorizontalPodAutoscaler", Namespace: hpa.Namespace, Name: hpa.Name})
+	}
+
+	pdb, err := r.PodDisruptionBudget()
+	if err != nil {
+		return nil, err
+	}
+	if pdb != nil {
+		refs = append(refs, ManagedProxyResourceRef{Kind: "PodDisruptionBudget", Namespace: pdb.Namespace, Name: pdb.Name})
+	}
+
+	return refs, nil
+}
+
 func (r *ResourceRender) ownerReferences() []metav1.OwnerReference {
 	var ownerReferences []metav1.OwnerReference
 	if r.ownerReferenceUID != nil {

internal/infrastructure/kubernetes/proxy_infra_test.go

@@ -7,6 +7,7 @@ package kubernetes
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -282,6 +283,90 @@ func TestDeleteProxyInfra(t *testing.T) {
 	}
 }
 
+func TestCreateProxyInfraDeletesTrackedServiceAccountsWhenListIsStale(t *testing.T) {
+	listStaleInterceptor := interceptor.Funcs{
+		Patch: interceptorFunc.Patch,
+		List: func(ctx context.Context, clnt client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
+			if err := clnt.List(ctx, list, opts...); err != nil {
+				return err
+			}
+			saList, ok := list.(*corev1.ServiceAccountList)
+			if ok && len(saList.Items) > 1 {
+				saList.Items = saList.Items[:1]
+			}
+			return nil
+		},
+	}
+
+	cli := fakeclient.NewClientBuilder().
+		WithScheme(envoygateway.GetScheme()).
+		WithInterceptorFuncs(listStaleInterceptor).
+		Build()
+	kube := newTestInfraWithClient(t, cli)
+	kube.EnvoyGateway.Provider = &egv1a1.EnvoyGatewayProvider{
+		Type: egv1a1.ProviderTypeKubernetes,
+		Kubernetes: &egv1a1.EnvoyGatewayKubernetesProvider{
+			Deploy: &egv1a1.KubernetesDeployMode{
+				Type: new(egv1a1.KubernetesDeployModeTypeGatewayNamespace),
+			},
+		},
+	}
+
+	ctx := context.Background()
+	require.NoError(t, setupOwnerReferenceResources(ctx, kube.Client))
+
+	gwNN := types.NamespacedName{Namespace: "ns1", Name: "gateway-1"}
+
+	defaultInfra := newProxyInfraForGateway(gwNN, "")
+	require.NoError(t, kube.CreateOrUpdateProxyInfra(ctx, defaultInfra))
+	assertTrackingConfigMap(t, ctx, kube, defaultInfra, []proxy.ManagedProxyResourceRef{
+		{Kind: "ServiceAccount", Namespace: gwNN.Namespace, Name: gwNN.Name},
+		{Kind: "Service", Namespace: gwNN.Namespace, Name: gwNN.Name},
+		{Kind: "Deployment", Namespace: gwNN.Namespace, Name: gwNN.Name},
+	})
+
+	customAInfra := newProxyInfraForGateway(gwNN, "custom-a-sa")
+	require.NoError(t, kube.CreateOrUpdateProxyInfra(ctx, customAInfra))
+	assertTrackingConfigMap(t, ctx, kube, customAInfra, []proxy.ManagedProxyResourceRef{
+		{Kind: "ServiceAccount", Namespace: gwNN.Namespace, Name: "custom-a-sa"},
+		{Kind: "Service", Namespace: gwNN.Namespace, Name: gwNN.Name},
+		{Kind: "Deployment", Namespace: gwNN.Namespace, Name: gwNN.Name},
+	})
+	require.NoError(t, kube.Client.Get(ctx, client.ObjectKey{Namespace: gwNN.Namespace, Name: "custom-a-sa"}, &corev1.ServiceAccount{}))
+	require.True(t, kerrors.IsNotFound(kube.Client.Get(ctx, gwNN, &corev1.ServiceAccount{})))
+
+	customBInfra := newProxyInfraForGateway(gwNN, "custom-b-sa")
+	require.NoError(t, kube.CreateOrUpdateProxyInfra(ctx, customBInfra))
+	assertTrackingConfigMap(t, ctx, kube, customBInfra, []proxy.ManagedProxyResourceRef{
+		{Kind: "ServiceAccount", Namespace: gwNN.Namespace, Name: "custom-b-sa"},
+		{Kind: "Service", Namespace: gwNN.Namespace, Name: gwNN.Name},
+		{Kind: "Deployment", Namespace: gwNN.Namespace, Name: gwNN.Name},
+	})
+	require.NoError(t, kube.Client.Get(ctx, client.ObjectKey{Namespace: gwNN.Namespace, Name: "custom-b-sa"}, &corev1.ServiceAccount{}))
+	require.True(t, kerrors.IsNotFound(kube.Client.Get(ctx, client.ObjectKey{Namespace: gwNN.Namespace, Name: "custom-a-sa"}, &corev1.ServiceAccount{})))
+}
+
+func TestCreateProxyInfraTracksRefsInControllerNamespaceConfigMapForMergedGateways(t *testing.T) {
+	kube := newTestInfra(t)
+	ctx := context.Background()
+	require.NoError(t, setupOwnerReferenceResources(ctx, kube.Client))
+
+	infra := ir.NewInfra()
+	infra.Proxy.Metadata.Labels = proxy.EnvoyAppLabel()
+	infra.Proxy.Metadata.Labels[gatewayapi.OwningGatewayClassLabel] = testGatewayClass
+	infra.Proxy.Metadata.OwnerReference = &ir.ResourceMetadata{
+		Kind: resource.KindGatewayClass,
+		Name: testGatewayClass,
+	}
+
+	require.NoError(t, kube.CreateOrUpdateProxyInfra(ctx, infra))
+	assertTrackingConfigMap(t, ctx, kube, infra, []proxy.ManagedProxyResourceRef{
+		{Kind: "ServiceAccount", Namespace: kube.ControllerNamespace, Name: expectedName(infra.Proxy, false)},
+		{Kind: "Service", Namespace: kube.ControllerNamespace, Name: expectedName(infra.Proxy, false)},
+		{Kind: "Deployment", Namespace: kube.ControllerNamespace, Name: expectedName(infra.Proxy, false)},
+	})
+}
+
 // This function uses setup creating Resources for OwnerReference.
 // When the default case, ProxyInfra Get OwnerReference from GatewayClass.
 // When enable GatewayNamespace mode, ProxyInfra Get OwnerReference from Gateway.
@@ -304,3 +389,47 @@ func setupOwnerReferenceResources(ctx context.Context, client *InfraClient) erro
 	}
 	return client.Create(ctx, gw)
 }
+
+func newProxyInfraForGateway(gwNN types.NamespacedName, serviceAccountName string) *ir.Infra {
+	infra := ir.NewInfra()
+	infra.Proxy.Name = gwNN.Name
+	infra.Proxy.Namespace = gwNN.Namespace
+	infra.Proxy.Metadata.Labels = proxy.EnvoyAppLabel()
+	infra.Proxy.Metadata.Labels[gatewayapi.OwningGatewayNamespaceLabel] = gwNN.Namespace
+	infra.Proxy.Metadata.Labels[gatewayapi.OwningGatewayNameLabel] = gwNN.Name
+	infra.Proxy.Metadata.OwnerReference = &ir.ResourceMetadata{
+		Kind: resource.KindGateway,
+		Name: gwNN.Name,
+	}
+	infra.Proxy.Config = &egv1a1.EnvoyProxy{
+		Spec: egv1a1.EnvoyProxySpec{
+			Provider: &egv1a1.EnvoyProxyProvider{
+				Type:       egv1a1.EnvoyProxyProviderTypeKubernetes,
+				Kubernetes: egv1a1.DefaultEnvoyProxyKubeProvider(),
+			},
+		},
+	}
+	if serviceAccountName != "" {
+		infra.Proxy.Config.Spec.Provider.Kubernetes.EnvoyServiceAccount = &egv1a1.KubernetesServiceAccountSpec{
+			Name: &serviceAccountName,
+		}
+	}
+	return infra
+}
+
+func assertTrackingConfigMap(t *testing.T, ctx context.Context, kube *Infra, infra *ir.Infra, expected []proxy.ManagedProxyResourceRef) {
+	t.Helper()
+
+	render, err := proxy.NewResourceRender(ctx, kube, infra)
+	require.NoError(t, err)
+
+	cm := &corev1.ConfigMap{}
+	require.NoError(t, kube.APIReader.Get(ctx, client.ObjectKey{
+		Namespace: kube.ControllerNamespace,
+		Name:      render.TrackingConfigMapName(),
+	}, cm))
+
+	var actual []proxy.ManagedProxyResourceRef
+	require.NoError(t, json.Unmarshal([]byte(cm.Data[trackedProxyRefsDataKey]), &actual))
+	require.Equal(t, expected, actual)
+}