more logs & metrics

Dockerfile

@@ -39,6 +39,8 @@ WORKDIR /
 COPY --from=builder /workspace/manager .
 USER 65532:65532
 
+LABEL source_repository="https://github.com/sapcc/Reloader"
+
 # Port for metrics and probes
 EXPOSE 9090
 

internal/pkg/controller/controller.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stakater/Reloader/internal/pkg/options"
 	"github.com/stakater/Reloader/internal/pkg/util"
 	"github.com/stakater/Reloader/pkg/kube"
+	"golang.org/x/time/rate"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -68,7 +69,10 @@ func NewController(
 	})
 	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("reloader-%s", resource)})
 
-	queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any]())
+	queue := workqueue.NewTypedRateLimitingQueue(workqueue.NewTypedMaxOfRateLimiter(
+		workqueue.NewTypedItemExponentialFailureRateLimiter[any](5*time.Millisecond, 60*time.Second),
+		&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(100), 1000)},
+	))
 
 	optionsModifier := func(options *metav1.ListOptions) {
 		if resource == "namespaces" {

internal/pkg/handler/create.go

@@ -22,6 +22,7 @@ func (r ResourceCreatedHandler) Handle() error {
 		logrus.Errorf("Resource creation handler received nil resource")
 	} else {
 		config, _ := r.GetConfig()
+		logrus.Infof("Will create: %s/%s", config.Namespace, config.ResourceName)
 		// Send webhook
 		if options.WebhookUrl != "" {
 			return sendUpgradeWebhook(config, options.WebhookUrl)

internal/pkg/handler/delete.go

@@ -31,6 +31,7 @@ func (r ResourceDeleteHandler) Handle() error {
 		logrus.Errorf("Resource delete handler received nil resource")
 	} else {
 		config, _ := r.GetConfig()
+		logrus.Infof("Will delete: %s/%s", config.Namespace, config.ResourceName)
 		// Send webhook
 		if options.WebhookUrl != "" {
 			return sendUpgradeWebhook(config, options.WebhookUrl)

internal/pkg/handler/update.go

@@ -24,6 +24,7 @@ func (r ResourceUpdatedHandler) Handle() error {
 	} else {
 		config, oldSHAData := r.GetConfig()
 		if config.SHAValue != oldSHAData {
+			logrus.Infof("Will upgrade: %s/%s", config.Namespace, config.ResourceName)
 			// Send a webhook if update
 			if options.WebhookUrl != "" {
 				return sendUpgradeWebhook(config, options.WebhookUrl)

internal/pkg/handler/upgrade.go

@@ -10,6 +10,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/parnurzeal/gorequest"
 	"github.com/prometheus/client_golang/prometheus"
@@ -239,6 +240,7 @@ func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callb
 
 // PerformAction invokes the deployment if there is any change in configmap or secret data
 func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error {
+	startTime := time.Now()
 	items := upgradeFuncs.ItemsFunc(clients, config.Namespace)
 
 	for _, item := range items {
@@ -250,7 +252,8 @@ func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callba
 			return err
 		}
 	}
-
+	endTime := time.Now()
+	collectors.ActionTime.With(prometheus.Labels{"resourceType": upgradeFuncs.ResourceType}).Set(float64(endTime.Sub(startTime).Seconds()))
 	return nil
 }
 

internal/pkg/metrics/prometheus.go

@@ -18,6 +18,7 @@ type Collectors struct {
 	Added               *prometheus.CounterVec
 	Updated             *prometheus.CounterVec
 	Deleted             *prometheus.CounterVec
+	ActionTime          *prometheus.GaugeVec
 }
 
 func NewCollectors() Collectors {
@@ -121,6 +122,16 @@ func NewCollectors() Collectors {
 			"resource",
 		},
 	)
+	actionTime := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "reloader",
+			Name:      "action_time_seconds",
+			Help:      "Gauge for the time taken to perform actions in seconds.",
+		},
+		[]string{
+			"resource",
+		},
+	)
 
 	return Collectors{
 		Reloaded:            reloaded,
@@ -132,6 +143,7 @@ func NewCollectors() Collectors {
 		Added:               added,
 		Updated:             updated,
 		Deleted:             deleted,
+		ActionTime:          actionTime,
 	}
 }
 
@@ -150,6 +162,7 @@ func SetupPrometheusEndpoint() Collectors {
 	prometheus.MustRegister(collectors.Added)
 	prometheus.MustRegister(collectors.Updated)
 	prometheus.MustRegister(collectors.Deleted)
+	prometheus.MustRegister(collectors.ActionTime)
 
 	http.Handle("/metrics", promhttp.Handler())