feat: add runner event metrics (#8802)

* add metrics for runner

Signed-off-by: zirain <zirain2009@gmail.com>

* rename

Signed-off-by: zirain <zirain2009@gmail.com>

* rename

Signed-off-by: zirain <zirain2009@gmail.com>

* reuse

Signed-off-by: zirain <zirain2009@gmail.com>

---------

Signed-off-by: zirain <zirain2009@gmail.com>
Signed-off-by: Arko Dasgupta <arkodg@users.noreply.github.com>
Co-authored-by: Arko Dasgupta <arkodg@users.noreply.github.com>

Author: zirain

internal/gatewayapi/runner/runner.go

@@ -190,6 +190,8 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
 		r.Logger,
 		message.Metadata{Runner: r.Name(), Message: message.ProviderResourcesMessageName}, sub,
 		func(update message.Update[string, *resource.ControllerResourcesContext], errChan chan error) {
+			message.PublishRunnerEventMetric(r.Name(), update.Delete)
+
 			parentCtx := context.Background()
 			if update.Value != nil && update.Value.Context != nil {
 				parentCtx = update.Value.Context

internal/globalratelimit/runner/runner.go

@@ -145,6 +145,8 @@ func (r *Runner) translateFromSubscription(ctx context.Context, c <-chan watchab
 		r.Logger,
 		message.Metadata{Runner: r.Name(), Message: message.XDSIRMessageName}, c,
 		func(update message.Update[string, *message.XdsIRWithContext], errChan chan error) {
+			message.PublishRunnerEventMetric(r.Name(), update.Delete)
+
 			parentCtx := ctx
 			if update.Value != nil && update.Value.Context != nil {
 				parentCtx = update.Value.Context

internal/infrastructure/runner/runner.go

@@ -108,6 +108,7 @@ func (r *Runner) updateProxyInfraFromSubscription(ctx context.Context, sub <-cha
 			default:
 			}
 			r.Logger.Info("received an update", "key", update.Key, "delete", update.Delete)
+			message.PublishRunnerEventMetric(r.Name(), update.Delete)
 			val := update.Value
 
 			if update.Delete {

internal/message/metrics.go

@@ -34,6 +34,12 @@ var (
 		"Total number of published updates to watchable queue.",
 	)
 
-	runnerLabel  = metrics.NewLabel("runner")
-	messageLabel = metrics.NewLabel("message")
+	watchableEventTotal = metrics.NewCounter(
+		"watchable_event_total",
+		"Total number of runner events.",
+	)
+
+	runnerLabel          = metrics.NewLabel("runner")
+	messageLabel         = metrics.NewLabel("message")
+	runnerEventTypeLabel = metrics.NewLabel("event_type")
 )

internal/message/watchutil.go

@@ -27,6 +27,17 @@ func PublishMetric(meta Metadata, count int) {
 	watchablePublishTotal.WithSuccess(meta.LabelValues()...).Add(float64(count))
 }
 
+func PublishRunnerEventMetric(runnerName string, isDelete bool) {
+	eventType := "update"
+	if isDelete {
+		eventType = "delete"
+	}
+	watchableEventTotal.With(
+		runnerEventTypeLabel.Value(eventType),
+		runnerLabel.Value(runnerName),
+	).Add(1)
+}
+
 func (m Metadata) LabelValues() []metrics.LabelValue {
 	labels := make([]metrics.LabelValue, 0, 2)
 	if m.Runner != "" {

internal/xds/runner/runner.go

@@ -263,6 +263,8 @@ func (r *Runner) translateFromSubscription(sub <-chan watchable.Snapshot[string,
 		r.Logger,
 		message.Metadata{Runner: r.Name(), Message: message.XDSIRMessageName}, sub,
 		func(update message.Update[string, *message.XdsIRWithContext], errChan chan error) {
+			message.PublishRunnerEventMetric(r.Name(), update.Delete)
+
 			parentCtx := context.Background()
 			if update.Value != nil && update.Value.Context != nil {
 				parentCtx = update.Value.Context

release-notes/current.yaml

@@ -21,3 +21,8 @@ deprecations: |
 
 # Other notable changes not covered by the above sections.
 Other changes: |
+  Moved Envoy Gateway CRDs into a sub-chart to avoid the Helm release secret exceeding the 1MB size limit when adding new API fields. Upgrade/Install behavior is unchanged for users.
+  The maximum number of rules in a RateLimit policy is increased from 128 to 256.
+  The maximum number of JWT providers allowed in `SecurityPolicy.spec.jwt.providers` is increased from 4 to 16.
+  Added `runner_event_total` metric to track update and delete events in infrastructure and gateway API runners for improved observability.
+