Skip to content

Commit 9bf7090

Browse files
ppaluckippalucki
committed
open telemetry PoC - client/server and explict token latency
1 parent 779e526 commit 9bf7090

3 files changed

Lines changed: 204 additions & 44 deletions

File tree

‎microservices-connector/cmd/router/main.go‎

Lines changed: 136 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,18 @@ import (
3636

3737
mcv1alpha3 "github.com/opea-project/GenAIInfra/microservices-connector/api/v1alpha3"
3838
flag "github.com/spf13/pflag"
39+
40+
// Prometheus and opentelemetry imports
41+
"github.com/prometheus/client_golang/prometheus/promhttp"
42+
43+
"go.opentelemetry.io/otel"
44+
"go.opentelemetry.io/otel/exporters/prometheus"
45+
api "go.opentelemetry.io/otel/metric"
46+
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
47+
48+
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
49+
50+
"go.opentelemetry.io/otel/metric"
3951
)
4052

4153
var (
@@ -72,6 +84,73 @@ type ReadCloser struct {
7284
*bytes.Reader
7385
}
7486

87+
var (
88+
firstTokenLatencyMeasure metric.Float64Histogram
89+
nextTokenLatencyMeasure metric.Float64Histogram
90+
)
91+
92+
func init() {
93+
94+
// The exporter embeds a default OpenTelemetry Reader and
95+
// implements prometheus.Collector, allowing it to be used as
96+
// both a Reader and Collector.
97+
exporter, err := prometheus.New()
98+
if err != nil {
99+
log.Error(err, "metrics: cannot init prometheus collector")
100+
}
101+
provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter))
102+
otel.SetMeterProvider(provider)
103+
104+
// ppalucki: Own metrics defintion bellow
105+
const meterName = "entrag-telemetry"
106+
meter := provider.Meter(meterName)
107+
108+
// extraAttributes := api.WithAttributes(
109+
// attribute.Key("xxx").String("yyy"),
110+
// attribute.Key("zzz").String("vvv"),
111+
// )
112+
113+
// a) Example float64 counter
114+
//fooCounter, err := meter.Float64Counter("foo_float64_counter", api.WithDescription("some float64 counter"))
115+
//if err != nil {
116+
// log.Error(err, "metrics: cannot init foo simple counter")
117+
//}
118+
//// synchornous counter: modify counter and add attirbutes
119+
//ctx := context.Background()
120+
//fooCounter.Add(ctx, 15, extraAttributes)
121+
//
122+
//// b) Example asynchronous (obeservalbe) counter with callback (interal state of router)
123+
//fooObsGuage, err := meter.Float64ObservableGauge("bar_obs_floa64_guage", api.WithDescription("a fun little gauge"))
124+
//if err != nil {
125+
// log.Error(err, "metrics: cannot init guage")
126+
//}
127+
//_, err = meter.RegisterCallback(func(_ context.Context, o api.Observer) error {
128+
// o.ObserveFloat64(fooObsGuage, 7, extraAttributes)
129+
// return nil
130+
//}, fooObsGuage)
131+
//if err != nil {
132+
// log.Error(err, "metrics: cannot register callback for bar obs gauage")
133+
//}
134+
firstTokenLatencyMeasure, err = meter.Float64Histogram(
135+
"llm.first.token.latency",
136+
metric.WithUnit("ms"),
137+
metric.WithDescription("Measures the first token generate duration of."),
138+
api.WithExplicitBucketBoundaries(1, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16364),
139+
)
140+
if err != nil {
141+
log.Error(err, "metrics: cannot register first token histogram measure")
142+
}
143+
nextTokenLatencyMeasure, err = meter.Float64Histogram(
144+
"llm.next.token.latency",
145+
metric.WithUnit("ms"),
146+
metric.WithDescription("Measures the next token generate duration of."),
147+
api.WithExplicitBucketBoundaries(1, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16364),
148+
)
149+
if err != nil {
150+
log.Error(err, "metrics: cannot register next token histogram measure")
151+
}
152+
}
153+
75154
func (ReadCloser) Close() error {
76155
// Typically, you would release resources here, but for bytes.Reader, there's nothing to do.
77156
return nil
@@ -173,7 +252,22 @@ func callService(
173252
if val := req.Header.Get("Content-Type"); val == "" {
174253
req.Header.Add("Content-Type", "application/json")
175254
}
176-
resp, err := http.DefaultClient.Do(req)
255+
256+
// Wrap default Go Http client with opentelemetry transport to provide extra metrics/pass context/create spans:
257+
// - "http.client.request.size" - "Measures the size of HTTP request messages." // Outgoing request bytes total
258+
// - "http.client.response.size" - "Measures the size of HTTP response messages." // Outgoing response bytes total
259+
// - "http.client.duration" - "Measures the duration of outbound HTTP requests." // Outgoing end to end duration, milliseconds
260+
// TODO/traces: span will be created with "client" as default, as middleware it possbile shouldbe "internal"
261+
client := http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)}
262+
263+
// add test lablel !!!?
264+
// labeler := &otelhttp.Labeler{}
265+
// labeler.Add(attribute.String("foo", "testFoo"))
266+
// ctx := context.Background()
267+
// ctx = otelhttp.ContextWithLabeler(ctx, labeler)
268+
// req = req.WithContext(ctx)
269+
270+
resp, err := client.Do(req)
177271

178272
if err != nil {
179273
log.Error(err, "An error has occurred while calling service", "service", serviceUrl)
@@ -518,7 +612,14 @@ func routeStep(nodeName string,
518612
}
519613

520614
func mcGraphHandler(w http.ResponseWriter, req *http.Request) {
521-
ctx, cancel := context.WithTimeout(req.Context(), time.Minute)
615+
616+
// TODO: traces
617+
// span := trace.SpanFromContext(ctx)
618+
// bag := baggage.FromContext(ctx)
619+
// span.AddEvent("handling this...", trace.WithAttributes(uk.String(bag.Member("username").Value())))
620+
621+
_, _ = io.WriteString(w, "Hello, world!\n")
622+
ctx, cancel := context.WithTimeout(req.Context(), 3600*time.Minute)
522623
defer cancel()
523624

524625
done := make(chan struct{})
@@ -550,9 +651,22 @@ func mcGraphHandler(w http.ResponseWriter, req *http.Request) {
550651
}()
551652

552653
w.Header().Set("Content-Type", "application/json")
654+
firstTokenCollected := false
553655
buffer := make([]byte, BufferSize)
554656
for {
657+
658+
// measure time of reading another portion of response
659+
tokenStartTime := time.Now()
555660
n, err := responseBody.Read(buffer)
661+
elapsedTimeMilisecond := float64(time.Since(tokenStartTime)) / float64(time.Millisecond)
662+
663+
if !firstTokenCollected {
664+
firstTokenCollected = true
665+
firstTokenLatencyMeasure.Record(ctx, elapsedTimeMilisecond)
666+
} else {
667+
nextTokenLatencyMeasure.Record(ctx, elapsedTimeMilisecond)
668+
}
669+
556670
if err != nil && err != io.EOF {
557671
log.Error(err, "failed to read from response body")
558672
http.Error(w, "failed to read from response body", http.StatusInternalServerError)
@@ -728,8 +842,26 @@ func handleMultipartError(writer *multipart.Writer, err error) {
728842

729843
func initializeRoutes() *http.ServeMux {
730844
mux := http.NewServeMux()
731-
mux.HandleFunc("/", mcGraphHandler)
732-
mux.HandleFunc("/dataprep", mcDataHandler)
845+
846+
// Wrap connector handlers with otelhttp wrappers
847+
// "http.server.request.size" - Int64Counter - "Measures the size of HTTP request messages" (Incoming request bytes total)
848+
// "http.server.response.size" - Int64Counter - "Measures the size of HTTP response messages" (Incoming response bytes total)
849+
// "http.server.duration" - Float64histogram "Measures the duration of inbound HTTP requests." (Incoming end to end duration, milliseconds)
850+
handleFunc := func(pattern string, handlerFunc func(http.ResponseWriter, *http.Request), operation string) {
851+
handler := otelhttp.NewHandler(otelhttp.WithRouteTag(pattern, http.HandlerFunc(handlerFunc)), operation)
852+
mux.Handle(pattern, handler)
853+
}
854+
855+
handleFunc("/", mcGraphHandler, "mcGraphHandler")
856+
handleFunc("/dataprep", mcDataHandler, "mcDataHandler")
857+
858+
// TODO: what is option as a string?!?!
859+
//mux.Handle("/", otelhttp.NewHandler(http.HandlerFunc(mcGraphHandler), "mcGraphHandler"))
860+
//mux.Handle("/dataprep", otelhttp.NewHandler(http.HandlerFunc(mcDataHandler), "mcDataHandler"))
861+
862+
// Export metric using Prometheus client built-in handler.
863+
mux.Handle("/metrics", promhttp.Handler())
864+
733865
return mux
734866
}
735867

‎microservices-connector/go.mod‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,15 @@ require (
66
github.com/onsi/ginkgo/v2 v2.14.0
77
github.com/onsi/gomega v1.30.0
88
github.com/pkg/errors v0.9.1
9+
github.com/prometheus/client_golang v1.19.1
910
github.com/spf13/pflag v1.0.5
10-
github.com/stretchr/testify v1.8.4
11+
github.com/stretchr/testify v1.9.0
1112
github.com/tidwall/gjson v1.17.1
13+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0
14+
go.opentelemetry.io/otel v1.28.0
15+
go.opentelemetry.io/otel/exporters/prometheus v0.50.0
16+
go.opentelemetry.io/otel/metric v1.28.0
17+
go.opentelemetry.io/otel/sdk/metric v1.28.0
1218
k8s.io/api v0.29.2
1319
k8s.io/apimachinery v0.29.2
1420
k8s.io/client-go v0.29.2
@@ -18,12 +24,14 @@ require (
1824

1925
require (
2026
github.com/beorn7/perks v1.0.1 // indirect
21-
github.com/cespare/xxhash/v2 v2.2.0 // indirect
27+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
2228
github.com/davecgh/go-spew v1.1.1 // indirect
2329
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
2430
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
31+
github.com/felixge/httpsnoop v1.0.4 // indirect
2532
github.com/fsnotify/fsnotify v1.7.0 // indirect
26-
github.com/go-logr/logr v1.4.1 // indirect
33+
github.com/go-logr/logr v1.4.2 // indirect
34+
github.com/go-logr/stdr v1.2.2 // indirect
2735
github.com/go-logr/zapr v1.3.0 // indirect
2836
github.com/go-openapi/jsonpointer v0.19.6 // indirect
2937
github.com/go-openapi/jsonreference v0.20.2 // indirect
@@ -45,24 +53,25 @@ require (
4553
github.com/modern-go/reflect2 v1.0.2 // indirect
4654
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
4755
github.com/pmezard/go-difflib v1.0.0 // indirect
48-
github.com/prometheus/client_golang v1.19.1 // indirect
49-
github.com/prometheus/client_model v0.6.0 // indirect
50-
github.com/prometheus/common v0.53.0 // indirect
51-
github.com/prometheus/procfs v0.12.0 // indirect
56+
github.com/prometheus/client_model v0.6.1 // indirect
57+
github.com/prometheus/common v0.55.0 // indirect
58+
github.com/prometheus/procfs v0.15.1 // indirect
5259
github.com/tidwall/match v1.1.1 // indirect
5360
github.com/tidwall/pretty v1.2.0 // indirect
61+
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
62+
go.opentelemetry.io/otel/trace v1.28.0 // indirect
5463
go.uber.org/multierr v1.11.0 // indirect
5564
go.uber.org/zap v1.27.0 // indirect
5665
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
57-
golang.org/x/net v0.25.0 // indirect
58-
golang.org/x/oauth2 v0.20.0 // indirect
59-
golang.org/x/sys v0.20.0 // indirect
60-
golang.org/x/term v0.20.0 // indirect
61-
golang.org/x/text v0.15.0 // indirect
66+
golang.org/x/net v0.26.0 // indirect
67+
golang.org/x/oauth2 v0.21.0 // indirect
68+
golang.org/x/sys v0.21.0 // indirect
69+
golang.org/x/term v0.21.0 // indirect
70+
golang.org/x/text v0.16.0 // indirect
6271
golang.org/x/time v0.5.0 // indirect
63-
golang.org/x/tools v0.21.0 // indirect
72+
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
6473
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
65-
google.golang.org/protobuf v1.34.1 // indirect
74+
google.golang.org/protobuf v1.34.2 // indirect
6675
gopkg.in/inf.v0 v0.9.1 // indirect
6776
gopkg.in/yaml.v2 v2.4.0 // indirect
6877
gopkg.in/yaml.v3 v3.0.1 // indirect

0 commit comments

Comments
 (0)