Skip to content

Commit 5eb7a97

Browse files
iakmciakmc
authored
Add custom Prometheus metrics (#636)
* Add custom Prometheus metrics Signed-off-by: Iskren <iskren@kubermatic.com> more metrics Signed-off-by: Iskren <iskren@kubermatic.com> go mod tidy required to pass the tests Signed-off-by: Iskren <iskren@kubermatic.com> * Remove redundant ReconcileDuration metric Signed-off-by: Iskren <iskren@kubermatic.com> --------- Signed-off-by: Iskren <iskren@kubermatic.com>
1 parent bd0685c commit 5eb7a97

11 files changed

Lines changed: 185 additions & 11 deletions

File tree

‎go.mod‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ require (
2626
github.com/opencontainers/go-digest v1.0.0
2727
github.com/platform-mesh/golang-commons v0.17.1
2828
github.com/platform-mesh/subroutines v0.4.3
29+
github.com/prometheus/client_golang v1.23.2
2930
github.com/rs/zerolog v1.35.1
3031
github.com/spf13/cobra v1.10.2
3132
github.com/spf13/pflag v1.0.10
@@ -88,6 +89,7 @@ require (
8889
github.com/kcp-dev/apimachinery/v2 v2.31.1 // indirect
8990
github.com/kcp-dev/logicalcluster/v3 v3.0.5 // indirect
9091
github.com/klauspost/compress v1.18.1 // indirect
92+
github.com/kylelemons/godebug v1.1.0 // indirect
9193
github.com/mattn/go-colorable v0.1.14 // indirect
9294
github.com/mattn/go-isatty v0.0.21 // indirect
9395
github.com/mitchellh/reflectwalk v1.0.2 // indirect
@@ -98,7 +100,6 @@ require (
98100
github.com/onsi/gomega v1.39.1 // indirect
99101
github.com/pkg/errors v0.9.1 // indirect
100102
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
101-
github.com/prometheus/client_golang v1.23.2 // indirect
102103
github.com/prometheus/client_model v0.6.2 // indirect
103104
github.com/prometheus/common v0.67.2 // indirect
104105
github.com/prometheus/procfs v0.20.1 // indirect

‎internal/controller/platformmesh_controller.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import (
4141

4242
corev1alpha1 "github.com/platform-mesh/platform-mesh-operator/api/v1alpha1"
4343
"github.com/platform-mesh/platform-mesh-operator/internal/config"
44+
"github.com/platform-mesh/platform-mesh-operator/internal/metrics"
4445
pmsubs "github.com/platform-mesh/platform-mesh-operator/pkg/subroutines"
4546
)
4647

@@ -61,7 +62,13 @@ type PlatformMeshReconciler struct {
6162
// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
6263

6364
func (r *PlatformMeshReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) {
64-
return r.lifecycle.Reconcile(ctx, req)
65+
result, err := r.lifecycle.Reconcile(ctx, req)
66+
labelResult := "success"
67+
if err != nil {
68+
labelResult = "error"
69+
}
70+
metrics.ReconcileTotal.WithLabelValues(pmReconcilerName, labelResult).Inc()
71+
return result, err
6572
}
6673

6774
// SetupWithManager sets up the controller with the Manager.

‎internal/controller/resource_controller.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737
mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile"
3838

3939
"github.com/platform-mesh/platform-mesh-operator/internal/config"
40+
"github.com/platform-mesh/platform-mesh-operator/internal/metrics"
4041
pmsubs "github.com/platform-mesh/platform-mesh-operator/pkg/subroutines"
4142
"github.com/platform-mesh/platform-mesh-operator/pkg/subroutines/resource"
4243
)
@@ -58,7 +59,13 @@ type ResourceReconciler struct {
5859
}
5960

6061
func (r *ResourceReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) {
61-
return r.lifecycle.Reconcile(ctx, req)
62+
result, err := r.lifecycle.Reconcile(ctx, req)
63+
labelResult := "success"
64+
if err != nil {
65+
labelResult = "error"
66+
}
67+
metrics.ReconcileTotal.WithLabelValues(resourceReconcilerName, labelResult).Inc()
68+
return result, err
6269
}
6370

6471
// SetupWithManager sets up the controller with the Manager.

‎internal/metrics/metrics.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package metrics
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
6+
)
7+
8+
var (
9+
// ReconcileTotal counts reconcile calls per controller and result (success/error).
10+
ReconcileTotal = prometheus.NewCounterVec(
11+
prometheus.CounterOpts{
12+
Name: "platform_mesh_operator_reconcile_total",
13+
Help: "Total number of reconcile calls by controller and result.",
14+
},
15+
[]string{"controller", "result"},
16+
)
17+
18+
// SubroutineTotal counts Process calls per subroutine and result (success/error).
19+
SubroutineTotal = prometheus.NewCounterVec(
20+
prometheus.CounterOpts{
21+
Name: "platform_mesh_operator_subroutine_total",
22+
Help: "Total number of subroutine Process calls by subroutine and result.",
23+
},
24+
[]string{"subroutine", "result"},
25+
)
26+
27+
// SubroutineDuration observes how long each subroutine Process call takes.
28+
SubroutineDuration = prometheus.NewHistogramVec(
29+
prometheus.HistogramOpts{
30+
Name: "platform_mesh_operator_subroutine_duration_seconds",
31+
Help: "Duration of subroutine Process calls in seconds by subroutine.",
32+
Buckets: prometheus.DefBuckets,
33+
},
34+
[]string{"subroutine"},
35+
)
36+
)
37+
38+
func init() {
39+
ctrlmetrics.Registry.MustRegister(
40+
ReconcileTotal,
41+
SubroutineTotal,
42+
SubroutineDuration,
43+
)
44+
}

‎internal/metrics/metrics_test.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package metrics_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/prometheus/client_golang/prometheus/testutil"
7+
"github.com/stretchr/testify/suite"
8+
9+
"github.com/platform-mesh/platform-mesh-operator/internal/metrics"
10+
)
11+
12+
type MetricsTestSuite struct {
13+
suite.Suite
14+
}
15+
16+
func TestMetricsTestSuite(t *testing.T) {
17+
suite.Run(t, new(MetricsTestSuite))
18+
}
19+
20+
// TestReconcileTotal verifies that the ReconcileTotal counter increments
21+
// correctly for each controller/result label combination.
22+
func (s *MetricsTestSuite) TestReconcileTotal() {
23+
before := testutil.ToFloat64(metrics.ReconcileTotal.WithLabelValues("PlatformMeshReconciler", "success"))
24+
metrics.ReconcileTotal.WithLabelValues("PlatformMeshReconciler", "success").Inc()
25+
s.Require().Equal(before+1, testutil.ToFloat64(metrics.ReconcileTotal.WithLabelValues("PlatformMeshReconciler", "success")))
26+
27+
before = testutil.ToFloat64(metrics.ReconcileTotal.WithLabelValues("ResourceReconciler", "error"))
28+
metrics.ReconcileTotal.WithLabelValues("ResourceReconciler", "error").Inc()
29+
s.Require().Equal(before+1, testutil.ToFloat64(metrics.ReconcileTotal.WithLabelValues("ResourceReconciler", "error")))
30+
}
31+
32+
// TestSubroutineTotal verifies that the SubroutineTotal counter increments
33+
// correctly for each subroutine/result label combination.
34+
func (s *MetricsTestSuite) TestSubroutineTotal() {
35+
before := testutil.ToFloat64(metrics.SubroutineTotal.WithLabelValues("deployment", "success"))
36+
metrics.SubroutineTotal.WithLabelValues("deployment", "success").Inc()
37+
s.Require().Equal(before+1, testutil.ToFloat64(metrics.SubroutineTotal.WithLabelValues("deployment", "success")))
38+
39+
before = testutil.ToFloat64(metrics.SubroutineTotal.WithLabelValues("kcpsetup", "error"))
40+
metrics.SubroutineTotal.WithLabelValues("kcpsetup", "error").Inc()
41+
s.Require().Equal(before+1, testutil.ToFloat64(metrics.SubroutineTotal.WithLabelValues("kcpsetup", "error")))
42+
}
43+
44+
// TestSubroutineDuration verifies that the SubroutineDuration histogram records
45+
// observations per subroutine label.
46+
func (s *MetricsTestSuite) TestSubroutineDuration() {
47+
before := testutil.CollectAndCount(metrics.SubroutineDuration)
48+
metrics.SubroutineDuration.WithLabelValues("deployment").Observe(0.1)
49+
s.Assert().Greater(testutil.CollectAndCount(metrics.SubroutineDuration), before)
50+
}

‎pkg/subroutines/deployment.go‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"path/filepath"
1010
"strings"
1111
"text/template"
12+
"time"
1213

1314
pmconfig "github.com/platform-mesh/golang-commons/config"
1415
"github.com/platform-mesh/golang-commons/errors"
@@ -29,6 +30,7 @@ import (
2930

3031
"github.com/platform-mesh/platform-mesh-operator/api/v1alpha1"
3132
"github.com/platform-mesh/platform-mesh-operator/internal/config"
33+
"github.com/platform-mesh/platform-mesh-operator/internal/metrics"
3234
"github.com/platform-mesh/platform-mesh-operator/pkg/merge"
3335
)
3436

@@ -167,7 +169,16 @@ func (r *DeploymentSubroutine) Finalizers(instance client.Object) []string { //
167169
return []string{}
168170
}
169171

170-
func (r *DeploymentSubroutine) Process(ctx context.Context, runtimeObj client.Object) (subroutines.Result, error) {
172+
func (r *DeploymentSubroutine) Process(ctx context.Context, runtimeObj client.Object) (res subroutines.Result, err error) {
173+
start := time.Now()
174+
defer func() {
175+
labelResult := "success"
176+
if err != nil {
177+
labelResult = "error"
178+
}
179+
metrics.SubroutineTotal.WithLabelValues(r.GetName(), labelResult).Inc()
180+
metrics.SubroutineDuration.WithLabelValues(r.GetName()).Observe(time.Since(start).Seconds())
181+
}()
171182
inst := runtimeObj.(*v1alpha1.PlatformMesh)
172183
log := logger.LoadLoggerFromContext(ctx).ChildLogger("subroutine", r.GetName())
173184
operatorCfg := pmconfig.LoadConfigFromContext(ctx).(config.OperatorConfig)

‎pkg/subroutines/featuretoggles.go‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/base64"
66
"fmt"
77
"path/filepath"
8+
"time"
89

910
pmconfig "github.com/platform-mesh/golang-commons/config"
1011
gcerrors "github.com/platform-mesh/golang-commons/errors"
@@ -18,6 +19,7 @@ import (
1819

1920
corev1alpha1 "github.com/platform-mesh/platform-mesh-operator/api/v1alpha1"
2021
"github.com/platform-mesh/platform-mesh-operator/internal/config"
22+
"github.com/platform-mesh/platform-mesh-operator/internal/metrics"
2123
)
2224

2325
const FeatureToggleSubroutineName = "FeatureToggleSubroutine"
@@ -62,7 +64,16 @@ func (r *FeatureToggleSubroutine) Finalizers(instance client.Object) []string {
6264
return []string{}
6365
}
6466

65-
func (r *FeatureToggleSubroutine) Process(ctx context.Context, runtimeObj client.Object) (subroutines.Result, error) {
67+
func (r *FeatureToggleSubroutine) Process(ctx context.Context, runtimeObj client.Object) (res subroutines.Result, err error) {
68+
start := time.Now()
69+
defer func() {
70+
labelResult := "success"
71+
if err != nil {
72+
labelResult = "error"
73+
}
74+
metrics.SubroutineTotal.WithLabelValues(r.GetName(), labelResult).Inc()
75+
metrics.SubroutineDuration.WithLabelValues(r.GetName()).Observe(time.Since(start).Seconds())
76+
}()
6677
log := logger.LoadLoggerFromContext(ctx).ChildLogger("subroutine", r.GetName())
6778
operatorCfg := pmconfig.LoadConfigFromContext(ctx).(config.OperatorConfig)
6879

‎pkg/subroutines/kcpsetup.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/base64"
66
"fmt"
77
"strings"
8+
"time"
89

910
pmconfig "github.com/platform-mesh/golang-commons/config"
1011
gcerrors "github.com/platform-mesh/golang-commons/errors"
@@ -19,6 +20,7 @@ import (
1920

2021
corev1alpha1 "github.com/platform-mesh/platform-mesh-operator/api/v1alpha1"
2122
"github.com/platform-mesh/platform-mesh-operator/internal/config"
23+
"github.com/platform-mesh/platform-mesh-operator/internal/metrics"
2224

2325
kcpapiv1alpha "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1"
2426
kcptenancyv1alpha "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1"
@@ -68,7 +70,16 @@ func (r *KcpsetupSubroutine) Finalizers(_ client.Object) []string { // coverage-
6870
return []string{KcpsetupSubroutineFinalizer}
6971
}
7072

71-
func (r *KcpsetupSubroutine) Process(ctx context.Context, runtimeObj client.Object) (subroutines.Result, error) {
73+
func (r *KcpsetupSubroutine) Process(ctx context.Context, runtimeObj client.Object) (res subroutines.Result, err error) {
74+
start := time.Now()
75+
defer func() {
76+
labelResult := "success"
77+
if err != nil {
78+
labelResult = "error"
79+
}
80+
metrics.SubroutineTotal.WithLabelValues(r.GetName(), labelResult).Inc()
81+
metrics.SubroutineDuration.WithLabelValues(r.GetName()).Observe(time.Since(start).Seconds())
82+
}()
7283
log := logger.LoadLoggerFromContext(ctx).ChildLogger("subroutine", r.GetName())
7384
operatorCfg := pmconfig.LoadConfigFromContext(ctx).(config.OperatorConfig)
7485

@@ -78,7 +89,7 @@ func (r *KcpsetupSubroutine) Process(ctx context.Context, runtimeObj client.Obje
7889
rootShard := &unstructured.Unstructured{}
7990
rootShard.SetGroupVersionKind(schema.GroupVersionKind{Group: "operator.kcp.io", Version: "v1alpha1", Kind: "RootShard"})
8091
// Wait for root shard to be ready
81-
err := r.client.Get(ctx, types.NamespacedName{Name: operatorCfg.KCP.RootShardName, Namespace: operatorCfg.KCP.Namespace}, rootShard)
92+
err = r.client.Get(ctx, types.NamespacedName{Name: operatorCfg.KCP.RootShardName, Namespace: operatorCfg.KCP.Namespace}, rootShard)
8293
if err != nil || !matchesConditionWithStatus(rootShard, "Available", "True") {
8394
log.Info().Msg("RootShard is not ready..")
8495
return subroutines.StopWithRequeue(DefaultRequeueInterval, "RootShard is not ready"), nil

‎pkg/subroutines/providersecret.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"net/url"
77
"path"
8+
"time"
89

910
pmconfig "github.com/platform-mesh/golang-commons/config"
1011
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -28,6 +29,7 @@ import (
2829

2930
corev1alpha1 "github.com/platform-mesh/platform-mesh-operator/api/v1alpha1"
3031
"github.com/platform-mesh/platform-mesh-operator/internal/config"
32+
"github.com/platform-mesh/platform-mesh-operator/internal/metrics"
3133
)
3234

3335
// HelmGetter is an interface for getting Helm releases
@@ -79,7 +81,16 @@ func (r *ProvidersecretSubroutine) Finalize(
7981

8082
func (r *ProvidersecretSubroutine) Process(
8183
ctx context.Context, runtimeObj client.Object,
82-
) (subroutines.Result, error) {
84+
) (res subroutines.Result, err error) {
85+
start := time.Now()
86+
defer func() {
87+
labelResult := "success"
88+
if err != nil {
89+
labelResult = "error"
90+
}
91+
metrics.SubroutineTotal.WithLabelValues(r.GetName(), labelResult).Inc()
92+
metrics.SubroutineDuration.WithLabelValues(r.GetName()).Observe(time.Since(start).Seconds())
93+
}()
8394
operatorCfg := pmconfig.LoadConfigFromContext(ctx).(config.OperatorConfig)
8495

8596
scheme := r.client.Scheme()
@@ -94,7 +105,7 @@ func (r *ProvidersecretSubroutine) Process(
94105
rootShard := &unstructured.Unstructured{}
95106
rootShard.SetGroupVersionKind(schema.GroupVersionKind{Group: "operator.kcp.io", Version: "v1alpha1", Kind: "RootShard"})
96107
// Wait for root shard to be ready
97-
err := r.client.Get(ctx, types.NamespacedName{Name: operatorCfg.KCP.RootShardName, Namespace: operatorCfg.KCP.Namespace}, rootShard)
108+
err = r.client.Get(ctx, types.NamespacedName{Name: operatorCfg.KCP.RootShardName, Namespace: operatorCfg.KCP.Namespace}, rootShard)
98109
if err != nil || !matchesConditionWithStatus(rootShard, "Available", "True") {
99110
log.Info().Msg("RootShard is not ready..")
100111
return subroutines.StopWithRequeue(DefaultRequeueInterval, "RootShard is not ready"), nil

‎pkg/subroutines/resource/subroutine.go‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717

1818
"github.com/platform-mesh/platform-mesh-operator/api/v1alpha1"
1919
"github.com/platform-mesh/platform-mesh-operator/internal/config"
20+
"github.com/platform-mesh/platform-mesh-operator/internal/metrics"
2021
"github.com/platform-mesh/platform-mesh-operator/pkg/ocm"
2122
"github.com/platform-mesh/platform-mesh-operator/pkg/subroutines"
2223
)
@@ -107,7 +108,16 @@ func getMetadataValue(obj *unstructured.Unstructured, key string) string {
107108
return ""
108109
}
109110

110-
func (r *ResourceSubroutine) Process(ctx context.Context, runtimeObj client.Object) (subroutineslib.Result, error) {
111+
func (r *ResourceSubroutine) Process(ctx context.Context, runtimeObj client.Object) (res subroutineslib.Result, err error) {
112+
start := time.Now()
113+
defer func() {
114+
labelResult := "success"
115+
if err != nil {
116+
labelResult = "error"
117+
}
118+
metrics.SubroutineTotal.WithLabelValues(r.GetName(), labelResult).Inc()
119+
metrics.SubroutineDuration.WithLabelValues(r.GetName()).Observe(time.Since(start).Seconds())
120+
}()
111121
inst := runtimeObj.(*unstructured.Unstructured)
112122
log := logger.LoadLoggerFromContext(ctx).ChildLogger("name", r.GetName())
113123

0 commit comments

Comments
 (0)