diff --git a/Dockerfile b/Dockerfile index 0b4876c38..4a1fdae82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,8 @@ WORKDIR / COPY --from=builder /workspace/manager . USER 65532:65532 +LABEL source_repository="https://github.com/sapcc/Reloader" + # Port for metrics and probes EXPOSE 9090 diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index 9ac9897af..b1fb850e4 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -11,6 +11,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/options" "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" + "golang.org/x/time/rate" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -68,7 +69,10 @@ func NewController( }) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("reloader-%s", resource)}) - queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any]()) + queue := workqueue.NewTypedRateLimitingQueue(workqueue.NewTypedMaxOfRateLimiter( + workqueue.NewTypedItemExponentialFailureRateLimiter[any](5*time.Millisecond, 60*time.Second), + &workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(100), 1000)}, + )) optionsModifier := func(options *metav1.ListOptions) { if resource == "namespaces" { diff --git a/internal/pkg/handler/create.go b/internal/pkg/handler/create.go index a35da5e7f..e8b944bf1 100644 --- a/internal/pkg/handler/create.go +++ b/internal/pkg/handler/create.go @@ -22,6 +22,7 @@ func (r ResourceCreatedHandler) Handle() error { logrus.Errorf("Resource creation handler received nil resource") } else { config, _ := r.GetConfig() + logrus.Infof("Will create: %s/%s", config.Namespace, config.ResourceName) // Send webhook if options.WebhookUrl != "" { return sendUpgradeWebhook(config, options.WebhookUrl) diff --git a/internal/pkg/handler/delete.go b/internal/pkg/handler/delete.go index 772cfcadb..e19d3b6d0 100644 --- a/internal/pkg/handler/delete.go +++ b/internal/pkg/handler/delete.go @@ -31,6 +31,7 @@ func (r ResourceDeleteHandler) Handle() error { logrus.Errorf("Resource delete handler received nil resource") } else { config, _ := r.GetConfig() + logrus.Infof("Will delete: %s/%s", config.Namespace, config.ResourceName) // Send webhook if options.WebhookUrl != "" { return sendUpgradeWebhook(config, options.WebhookUrl) diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index 0575e1901..afbb80832 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -24,6 +24,7 @@ func (r ResourceUpdatedHandler) Handle() error { } else { config, oldSHAData := r.GetConfig() if config.SHAValue != oldSHAData { + logrus.Infof("Will upgrade: %s/%s", config.Namespace, config.ResourceName) // Send a webhook if update if options.WebhookUrl != "" { return sendUpgradeWebhook(config, options.WebhookUrl) diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 8f394f8d2..ee423ba10 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -10,6 +10,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/parnurzeal/gorequest" "github.com/prometheus/client_golang/prometheus" @@ -239,6 +240,7 @@ func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callb // PerformAction invokes the deployment if there is any change in configmap or secret data func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error { + startTime := time.Now() items := upgradeFuncs.ItemsFunc(clients, config.Namespace) for _, item := range items { @@ -250,7 +252,8 @@ func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callba return err } } - + endTime := time.Now() + collectors.ActionTime.With(prometheus.Labels{"resourceType": upgradeFuncs.ResourceType}).Set(float64(endTime.Sub(startTime).Seconds())) return nil } diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index e9a3e5e98..9006b2d4f 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -18,6 +18,7 @@ type Collectors struct { Added *prometheus.CounterVec Updated *prometheus.CounterVec Deleted *prometheus.CounterVec + ActionTime *prometheus.GaugeVec } func NewCollectors() Collectors { @@ -121,6 +122,16 @@ func NewCollectors() Collectors { "resource", }, ) + actionTime := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "reloader", + Name: "action_time_seconds", + Help: "Gauge for the time taken to perform actions in seconds.", + }, + []string{ + "resource", + }, + ) return Collectors{ Reloaded: reloaded, @@ -132,6 +143,7 @@ func NewCollectors() Collectors { Added: added, Updated: updated, Deleted: deleted, + ActionTime: actionTime, } } @@ -150,6 +162,7 @@ func SetupPrometheusEndpoint() Collectors { prometheus.MustRegister(collectors.Added) prometheus.MustRegister(collectors.Updated) prometheus.MustRegister(collectors.Deleted) + prometheus.MustRegister(collectors.ActionTime) http.Handle("/metrics", promhttp.Handler())