Skip to content

Commit 89b4d22

Browse files
krsnaSurajkrsnaSuraj
authored andcommitted
fix(atenet): propagate trace context through router and namespace span/metric attrs
Fixes trace detachment in the router where two issues broke the trace chain: 1. Primary: resumer.go's singleflight used context.Background() inside DoChan, so otelgrpc saw no parent span and ateapi handler spans appeared in a separate trace. Now propagates the caller's span context into the background context. 2. Secondary: the ext_proc HeadersResponse only rewrote :authority and never injected traceparent/tracestate into the header mutation. Now injects the trace context so the upstream worker receives the same trace even when Envoy tracing is not configured. 3. Span and metric attributes used bare keys (atespace, actor, etc.) instead of the ate.* convention established in #412. Renamed to ate.atespace, ate.actor_name, ate.target_addr, ate.actor_template_namespace, ate.actor_template_name, ate.outcome. Fixes #427
1 parent c1ab095 commit 89b4d22

4 files changed

Lines changed: 53 additions & 16 deletions

File tree

cmd/atenet/internal/router/extproc.go

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,10 @@ func (s *ExtProcServer) handleRequestHeaders(
147147
// Host is invalid, respond with 404.
148148
return nil, metadata, "", "", "", invalidHostErr(metadata.host, err)
149149
}
150+
span.SetAttributes(
151+
attribute.String("ate.atespace", atespace),
152+
attribute.String("ate.actor.id", actorName),
153+
)
150154

151155
slog.InfoContext(ctx, "ResumeActor", slog.String("atespace", atespace), slog.String("actor", actorName))
152156
actor, err := s.resumer.ResumeActor(ctx, atespace, actorName)
@@ -176,9 +180,17 @@ func (s *ExtProcServer) handleRequestHeaders(
176180

177181
slog.InfoContext(ctx, "Route ok", slog.String("actor", actorName), slog.String("targetAddr", targetAddr))
178182

179-
// Route by rewriting the :authority header.
183+
span.SetAttributes(
184+
attribute.String("ate.target_addr", targetAddr),
185+
attribute.String("ate.actor.template.namespace", tmplNs),
186+
attribute.String("ate.actor.template.name", tmplName),
187+
)
188+
189+
// Route by rewriting the :authority header and injecting trace context
190+
// so the upstream worker receives the same trace as the ingress path.
180191
mutation := &extprocv3.HeaderMutation{}
181192
addAuthorityMutation(targetAddr, mutation)
193+
injectTraceContext(ctx, mutation)
182194

183195
return &extprocv3.HeadersResponse{
184196
Response: &extprocv3.CommonResponse{
@@ -192,9 +204,9 @@ func (s *ExtProcServer) recordRouteDuration(ctx context.Context, d time.Duration
192204
return
193205
}
194206
s.routeDuration.Record(ctx, d.Seconds(), metric.WithAttributes(
195-
attribute.String("actor_template_namespace", tmplNs),
196-
attribute.String("actor_template_name", tmplName),
197-
attribute.String("outcome", outcome),
207+
attribute.String("ate.actor.template.namespace", tmplNs),
208+
attribute.String("ate.actor.template.name", tmplName),
209+
attribute.String("ate.outcome", outcome),
198210
))
199211
}
200212

cmd/atenet/internal/router/extproc_out.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,13 @@
1515
package router
1616

1717
import (
18+
"context"
19+
1820
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
1921
extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
2022
envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3"
23+
"go.opentelemetry.io/otel"
24+
"go.opentelemetry.io/otel/propagation"
2125
)
2226

2327
// reqError carries an HTTP-mappable status code and a client-safe message.
@@ -43,6 +47,22 @@ func addAuthorityMutation(auth string, mut *extproc.HeaderMutation) {
4347
)
4448
}
4549

50+
// injectTraceContext injects the trace context from ctx into the header mutation,
51+
// so that Envoy forwards traceparent/tracestate to the upstream worker and spans
52+
// are connected across the full request path.
53+
func injectTraceContext(ctx context.Context, mutation *extproc.HeaderMutation) {
54+
headers := make(map[string]string)
55+
otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(headers))
56+
for k, v := range headers {
57+
mutation.SetHeaders = append(mutation.SetHeaders, &corev3.HeaderValueOption{
58+
Header: &corev3.HeaderValue{
59+
Key: k,
60+
Value: v,
61+
},
62+
})
63+
}
64+
}
65+
4666
func immediateResponse(statusCode envoy_type.StatusCode, message string) *extproc.ProcessingResponse {
4767
return &extproc.ProcessingResponse{
4868
Response: &extproc.ProcessingResponse_ImmediateResponse{

cmd/atenet/internal/router/extproc_test.go

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -230,17 +230,19 @@ func TestExtProcHeadersEvaluation(t *testing.T) {
230230
}
231231

232232
mutation := res.Response.GetHeaderMutation()
233-
if len(mutation.GetSetHeaders()) != 1 {
234-
t.Fatalf("expected exactly one Header option set, found: %v", mutation.GetSetHeaders())
235-
}
236-
237-
headerOption := mutation.GetSetHeaders()[0]
238-
if strings.ToLower(headerOption.Header.Key) != ":authority" {
239-
t.Errorf("invalid resulting dynamic parameter key: %s", headerOption.Header.Key)
233+
headers := mutation.GetSetHeaders()
234+
var found bool
235+
for _, h := range headers {
236+
if strings.ToLower(h.Header.Key) == ":authority" {
237+
found = true
238+
if string(h.Header.RawValue) != tc.expectedTarget {
239+
t.Errorf("invalid destination mapping found: %s, expected: %s", h.Header.RawValue, tc.expectedTarget)
240+
}
241+
break
242+
}
240243
}
241-
242-
if string(headerOption.Header.RawValue) != tc.expectedTarget {
243-
t.Errorf("invalid destination mapping found: %s, expected: %s", headerOption.Header.RawValue, tc.expectedTarget)
244+
if !found {
245+
t.Errorf(":authority header not found in mutation, got: %v", headers)
244246
}
245247

246248
// Confirm that query logs recorded metric trace details

cmd/atenet/internal/router/resumer.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ func NewActorResumer(apiClient ateapipb.ControlClient) *ActorResumer {
4646
func (r *ActorResumer) ResumeActor(ctx context.Context, atespace, actorName string) (*ateapipb.Actor, error) {
4747
ctx, span := otel.Tracer(routerServiceName).Start(ctx, "ResumeActor",
4848
trace.WithAttributes(
49-
attribute.String("atespace", atespace),
50-
attribute.String("actor", actorName),
49+
attribute.String("ate.atespace", atespace),
50+
attribute.String("ate.actor.id", actorName),
5151
))
5252
defer span.End()
5353

@@ -57,6 +57,9 @@ func (r *ActorResumer) ResumeActor(ctx context.Context, atespace, actorName stri
5757
// resume operation continues running for Caller 2 and Caller 3 without failing.
5858
bgCtx, bgCancel := context.WithTimeout(context.Background(), 15*time.Second)
5959
defer bgCancel()
60+
// Propagate the caller's span context so the gRPC spans are children
61+
// of ResumeActor rather than appearing as a separate trace.
62+
bgCtx = trace.ContextWithSpanContext(bgCtx, trace.SpanContextFromContext(ctx))
6063

6164
backoff := wait.Backoff{
6265
Steps: 7,

0 commit comments

Comments
 (0)