diff --git a/ddtrace/tracer/span.go b/ddtrace/tracer/span.go index 220ead1b7b5..d7e01a5fdd9 100644 --- a/ddtrace/tracer/span.go +++ b/ddtrace/tracer/span.go @@ -34,6 +34,7 @@ import ( "github.com/DataDog/dd-trace-go/v2/internal/locking" "github.com/DataDog/dd-trace-go/v2/internal/locking/assert" "github.com/DataDog/dd-trace-go/v2/internal/log" + iof "github.com/DataDog/dd-trace-go/v2/internal/openfeature" "github.com/DataDog/dd-trace-go/v2/internal/samplernames" "github.com/DataDog/dd-trace-go/v2/internal/stacktrace" "github.com/DataDog/dd-trace-go/v2/internal/telemetry" @@ -949,6 +950,47 @@ func (s *Span) serializeSpanEvents() { s.meta.Set("events", string(b)) } +// recordFFEEvaluation records a feature flag evaluation for FFE span enrichment. +// Used by github.com/DataDog/dd-trace-go/v2/openfeature via go:linkname. +// +// Feature flag evaluations are collected while a request is in progress and +// copied onto the root span as tags when the span finishes. The temporary +// storage lives in internal/openfeature, but access to it must be coordinated +// with the span lifecycle here. +// +// The OpenFeature hook reaches this function instead of writing to the store +// directly because it cannot safely inspect or synchronize with Span.finished. +// Holding s.mu serializes recording an evaluation with Finish draining the +// stored evaluations. The s.finished check prevents recording evaluations after +// the span has already finished; Finish drains the stored evaluations exactly +// once while holding the same lock. +func recordFFEEvaluation(s *Span, eval *iof.FeatureFlagEvaluation) { + if s == nil || eval == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + + if s.finished { + return + } + iof.AddSpanEnrichment(s, eval) +} + +// serializeFFEEvaluations writes the accumulated OpenFeature evaluation tags onto +// the span and removes the pending FFE span enrichment from the temporary store. +// +checklocks:s.mu +func (s *Span) serializeFFEEvaluations() { + assert.RWMutexLocked(&s.mu) + enrichment := iof.DrainSpanEnrichment(s) + if enrichment == nil { + return + } + for tag, value := range enrichment.GetSpanTags() { + s.meta.Set(tag, value) + } +} + // Finish closes this Span (but not its children) providing the duration // of its part of the tracing session. func (s *Span) Finish(opts ...FinishOption) { @@ -1052,6 +1094,7 @@ func (s *Span) finish(finishTime int64) { s.serializeSpanLinksInMeta() s.serializeSpanEvents() + s.serializeFFEEvaluations() s.enrichServiceSource() if s.duration == 0 { diff --git a/internal/env/supported_configurations.gen.go b/internal/env/supported_configurations.gen.go index d6b263a4377..1b305958646 100644 --- a/internal/env/supported_configurations.gen.go +++ b/internal/env/supported_configurations.gen.go @@ -65,6 +65,7 @@ var SupportedConfigurations = map[string]struct{}{ "DD_DYNAMIC_INSTRUMENTATION_ENABLED": {}, "DD_ENV": {}, "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": {}, + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": {}, "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED": {}, "DD_EXTERNAL_ENV": {}, "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": {}, diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index f0049cbed4d..917911a7919 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -399,6 +399,13 @@ "default": "false" } ], + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": [ + { + "implementation": "A", + "type": "boolean", + "default": "false" + } + ], "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED": [ { "implementation": "B", diff --git a/internal/openfeature/span_enrichment.go b/internal/openfeature/span_enrichment.go new file mode 100644 index 00000000000..1ac06aa2aad --- /dev/null +++ b/internal/openfeature/span_enrichment.go @@ -0,0 +1,242 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package openfeature + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "slices" + "strings" + "sync" + + "github.com/DataDog/dd-trace-go/v2/internal/log" +) + +// This file implements temporary span enrichment for feature flag evaluations. +// Evaluations are collected while a traced request is in progress, stored here +// keyed by the owning span, and drained by the tracer when that span finishes. +// The drained enrichment is then encoded as span tags. +// +// The store is global rather than tracer-owned so evaluations survive tracer +// swaps between evaluation and span finish. The key is intentionally typed as +// any to avoid importing ddtrace/tracer and creating an import cycle; callers use +// *ddtrace/tracer.Span as the key. +// +// SpanEnrichment is not safe for concurrent use. Callers must synchronize calls +// to AddSpanEnrichment and DrainSpanEnrichment with the key owner's lifecycle +// lock. For tracer spans, ddtrace/tracer/span.go holds Span.mu and checks +// Span.finished before adding, then drains while holding the same lock during +// Finish. + +// FeatureFlagEvaluation represents a single feature flag evaluation to be +// recorded in span enrichment. +type FeatureFlagEvaluation struct { + FlagKey string + // SerialId is the optional serial ID from flag metadata. Nil for runtime defaults. + SerialID *uint32 + // Subject is the targeting key / subject of experiment. Non-empty only when evaluation + // should be logged (doLog=true). + Subject string + // DefaultValue is the default value used. Non-nil only when a runtime default was used. + DefaultValue any +} + +const ( + spanEnrichmentMaxSerialIDs = 200 + spanEnrichmentMaxSubjects = 10 + spanEnrichmentMaxSerialIDsPerSubject = 20 + spanEnrichmentMaxRuntimeDefaults = 5 + spanEnrichmentMaxDefaultValueLen = 64 +) + +// spanEnrichments stores pending span enrichments keyed by an owner +// object (*ddtrace/tracer.Span). +var spanEnrichments sync.Map + +// AddSpanEnrichment records eval in the global span enrichment store. +func AddSpanEnrichment(key any, eval *FeatureFlagEvaluation) { + if key == nil || eval == nil { + return + } + + // Two phase Load+LoadOrStore to avoid allocating a + // SpanEnrichment on the hot path (entry already present) that + // LoadOrStore would require. + var enrichment *SpanEnrichment + if actual, ok := spanEnrichments.Load(key); ok { + enrichment = actual.(*SpanEnrichment) + } else { + e, _ := spanEnrichments.LoadOrStore(key, newSpanEnrichment()) + enrichment = e.(*SpanEnrichment) + } + enrichment.addEvaluation(eval) +} + +// DrainSpanEnrichment removes and returns enrichment from the global span +// enrichment store. +func DrainSpanEnrichment(key any) *SpanEnrichment { + if key == nil { + return nil + } + v, ok := spanEnrichments.LoadAndDelete(key) + if !ok { + return nil + } + return v.(*SpanEnrichment) +} + +// SpanEnrichment accumulates feature flag evaluations to be encoded as span tags. +// +// SpanEnrichment is not safe for concurrent use on its own. Thread safety is +// provided externally: every caller must hold the owning span's mu +// (ddtrace/tracer.Span.mu) before calling any method. +type SpanEnrichment struct { + serialIDs map[uint32]struct{} + subjects map[string]map[uint32]struct{} + runtimeDefaults map[string]string +} + +func newSpanEnrichment() *SpanEnrichment { + return &SpanEnrichment{ + serialIDs: make(map[uint32]struct{}), + subjects: make(map[string]map[uint32]struct{}), + runtimeDefaults: make(map[string]string), + } +} + +// addEvaluation records a single feature flag evaluation. +// The caller must hold the owning span's mu. +func (se *SpanEnrichment) addEvaluation(eval *FeatureFlagEvaluation) { + if eval == nil { + return + } + + if eval.SerialID != nil { + se.addSerialID(*eval.SerialID) + if eval.Subject != "" { + se.addSubject(eval.Subject, *eval.SerialID) + } + } else if eval.DefaultValue != nil { + se.addRuntimeDefault(eval.FlagKey, eval.DefaultValue) + } +} + +// Must be called with the owning span's mu held. +func (se *SpanEnrichment) addSerialID(sid uint32) { + if _, exists := se.serialIDs[sid]; !exists { + if len(se.serialIDs) >= spanEnrichmentMaxSerialIDs { + log.Debug("openfeature: span enrichment: too many flag serial IDs, dropping (max %d)", spanEnrichmentMaxSerialIDs) + return + } + se.serialIDs[sid] = struct{}{} + } +} + +// Must be called with the owning span's mu held. +func (se *SpanEnrichment) addSubject(subject string, sid uint32) { + subjectIDs, ok := se.subjects[subject] + if !ok { + if len(se.subjects) >= spanEnrichmentMaxSubjects { + log.Debug("openfeature: span enrichment: too many targeting keys, dropping (max %d)", spanEnrichmentMaxSubjects) + return + } + subjectIDs = make(map[uint32]struct{}) + se.subjects[subject] = subjectIDs + } + if len(subjectIDs) >= spanEnrichmentMaxSerialIDsPerSubject { + log.Debug("openfeature: span enrichment: too many experiments for subject %q, dropping (max %d)", subject, spanEnrichmentMaxSerialIDsPerSubject) + return + } + subjectIDs[sid] = struct{}{} +} + +// Must be called with the owning span's mu held. +func (se *SpanEnrichment) addRuntimeDefault(flagKey string, defaultValue any) { + if _, exists := se.runtimeDefaults[flagKey]; exists { + return + } + if len(se.runtimeDefaults) >= spanEnrichmentMaxRuntimeDefaults { + log.Debug("openfeature: span enrichment: too many runtime defaults, dropping (max %d)", spanEnrichmentMaxRuntimeDefaults) + return + } + var valueStr string + if v, ok := defaultValue.(string); ok { + valueStr = v + } else if b, err := json.Marshal(defaultValue); err == nil { + valueStr = string(b) + } else { + log.Debug("openfeature: span enrichment: failed to marshal runtime default value for key %q: %v", flagKey, err.Error()) + return + } + if len(valueStr) > spanEnrichmentMaxDefaultValueLen { + log.Debug("openfeature: span enrichment: runtime default value for key %q exceeds max length (%d), truncating", flagKey, spanEnrichmentMaxDefaultValueLen) + valueStr = valueStr[:spanEnrichmentMaxDefaultValueLen] + } + se.runtimeDefaults[flagKey] = strings.ToValidUTF8(valueStr, "") +} + +// GetSpanTags returns span tags encoding accumulated feature flag evaluations. +// The caller must hold the owning span's mu. +func (se *SpanEnrichment) GetSpanTags() map[string]string { + tags := make(map[string]string, 3) + + if len(se.serialIDs) > 0 { + tags["ffe_flags_enc"] = encodeSerialIDs(se.serialIDs) + } + + if len(se.subjects) > 0 { + subjects := make(map[string]string, len(se.subjects)) + for key, ids := range se.subjects { + sum := sha256.Sum256([]byte(key)) + hashKey := hex.EncodeToString(sum[:]) + subjects[hashKey] = encodeSerialIDs(ids) + } + if b, err := json.Marshal(subjects); err == nil { + tags["ffe_subjects_enc"] = string(b) + } else { + log.Debug("openfeature: span enrichment: failed to marshal subjects: %v", err.Error()) + } + } + + if len(se.runtimeDefaults) > 0 { + defaults := se.runtimeDefaults + if b, err := json.Marshal(defaults); err == nil { + tags["ffe_runtime_defaults"] = string(b) + } else { + log.Debug("openfeature: span enrichment: failed to marshal runtime defaults: %v", err.Error()) + } + } + + return tags +} + +// Encode a set of serial ids using unsigned LEB128 delta encoding, wrapped in base64. +func encodeSerialIDs(ids map[uint32]struct{}) string { + seq := make([]uint32, 0, len(ids)) + for id := range ids { + seq = append(seq, id) + } + + slices.Sort(seq) + + b := make([]byte, 0, 5*len(seq)) // 5 bytes is absolute max to encode an id + var prevID uint32 = 0 + for _, id := range seq { + diff := id - prevID + prevID = id + + // Unsigned LEB128 encoding. + for diff > 0x7f { + b = append(b, byte((diff&0x7f)|0x80)) + diff >>= 7 + } + b = append(b, byte(diff&0x7f)) + } + + return base64.StdEncoding.EncodeToString(b) +} diff --git a/internal/openfeature/span_enrichment_test.go b/internal/openfeature/span_enrichment_test.go new file mode 100644 index 00000000000..835d7839d5d --- /dev/null +++ b/internal/openfeature/span_enrichment_test.go @@ -0,0 +1,191 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package openfeature + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSpanEnrichment_GetSpanTags(t *testing.T) { + se := newSpanEnrichment() + + sid42 := uint32(42) + sid101 := uint32(101) + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: "experiment-b", SerialID: &sid101}) + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: "experiment-a", SerialID: &sid42, Subject: "user-123"}) + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: "experiment-a-dup", SerialID: &sid42, Subject: "user-123"}) + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: "default-flag", DefaultValue: "default-val"}) + + tags := se.GetSpanTags() + assert.Equal(t, []uint32{42, 101}, decodeSerialIDs(t, tags["ffe_flags_enc"])) + + var subjects map[string]string + require.NoError(t, json.Unmarshal([]byte(tags["ffe_subjects_enc"]), &subjects)) + user123HashBytes := sha256.Sum256([]byte("user-123")) + user123Hash := hex.EncodeToString(user123HashBytes[:]) + assert.Equal(t, map[string]string{user123Hash: encodeSerialIDs(map[uint32]struct{}{42: {}})}, subjects) + + var defaults map[string]string + require.NoError(t, json.Unmarshal([]byte(tags["ffe_runtime_defaults"]), &defaults)) + assert.Equal(t, map[string]string{"default-flag": "default-val"}, defaults) +} + +func TestSpanEnrichment_OmitEmpty(t *testing.T) { + se := newSpanEnrichment() + tags := se.GetSpanTags() + assert.Empty(t, tags) +} + +func TestSpanEnrichment_Limits(t *testing.T) { + t.Run("serial ids", func(t *testing.T) { + se := newSpanEnrichment() + for i := range spanEnrichmentMaxSerialIDs + 1 { + sid := uint32(i + 1) + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: fmt.Sprintf("flag-%d", i), SerialID: &sid}) + } + + ids := decodeSerialIDs(t, se.GetSpanTags()["ffe_flags_enc"]) + require.Len(t, ids, spanEnrichmentMaxSerialIDs) + assert.Equal(t, uint32(spanEnrichmentMaxSerialIDs), ids[len(ids)-1]) + }) + + t.Run("subjects", func(t *testing.T) { + se := newSpanEnrichment() + for i := range spanEnrichmentMaxSubjects + 1 { + sid := uint32(i + 1) + se.addEvaluation(&FeatureFlagEvaluation{ + FlagKey: fmt.Sprintf("flag-%d", i), + SerialID: &sid, + Subject: fmt.Sprintf("user-%d", i), + }) + } + + var subjects map[string]string + require.NoError(t, json.Unmarshal([]byte(se.GetSpanTags()["ffe_subjects_enc"]), &subjects)) + require.Len(t, subjects, spanEnrichmentMaxSubjects) + }) + + t.Run("runtime defaults", func(t *testing.T) { + se := newSpanEnrichment() + for i := range spanEnrichmentMaxRuntimeDefaults + 1 { + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: fmt.Sprintf("flag-%d", i), DefaultValue: []int{i}}) + } + + var defaults map[string]string + require.NoError(t, json.Unmarshal([]byte(se.GetSpanTags()["ffe_runtime_defaults"]), &defaults)) + require.Len(t, defaults, spanEnrichmentMaxRuntimeDefaults) + }) + + t.Run("serial ids per subject", func(t *testing.T) { + se := newSpanEnrichment() + for i := range spanEnrichmentMaxSerialIDsPerSubject + 1 { + sid := uint32(i + 1) + se.addEvaluation(&FeatureFlagEvaluation{ + FlagKey: fmt.Sprintf("flag-%d", i), + SerialID: &sid, + Subject: "user-1", + }) + } + + var subjects map[string]string + require.NoError(t, json.Unmarshal([]byte(se.GetSpanTags()["ffe_subjects_enc"]), &subjects)) + require.Len(t, subjects, 1) + user1HashBytes := sha256.Sum256([]byte("user-1")) + user1Hash := hex.EncodeToString(user1HashBytes[:]) + ids := decodeSerialIDs(t, subjects[user1Hash]) + assert.Len(t, ids, spanEnrichmentMaxSerialIDsPerSubject) + }) +} + +func TestSpanEnrichment_NonStringRuntimeDefault(t *testing.T) { + se := newSpanEnrichment() + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: "int-slice-flag", DefaultValue: []int{1, 2, 3}}) + + var defaults map[string]string + require.NoError(t, json.Unmarshal([]byte(se.GetSpanTags()["ffe_runtime_defaults"]), &defaults)) + assert.Equal(t, "[1,2,3]", defaults["int-slice-flag"]) +} + +func TestSpanEnrichment_TruncateRuntimeDefault(t *testing.T) { + se := newSpanEnrichment() + longVal := strings.Repeat("a", spanEnrichmentMaxDefaultValueLen+10) + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: "long-flag", DefaultValue: longVal}) + + var defaults map[string]string + require.NoError(t, json.Unmarshal([]byte(se.GetSpanTags()["ffe_runtime_defaults"]), &defaults)) + got := defaults["long-flag"] + assert.Len(t, got, spanEnrichmentMaxDefaultValueLen) + assert.Equal(t, strings.Repeat("a", spanEnrichmentMaxDefaultValueLen), got) +} + +// TestSpanEnrichment_TruncateRuntimeDefault_UTF8 verifies that truncation never +// produces invalid UTF-8 when the cut point falls inside a multi-byte sequence. +func TestSpanEnrichment_TruncateRuntimeDefault_UTF8(t *testing.T) { + // Build a string whose multi-byte character (€ = 3 bytes: \xe2\x82\xac) + // straddles the truncation boundary: 62 ASCII bytes + "€" = 65 bytes total. + // Byte-slicing at spanEnrichmentMaxDefaultValueLen (64) would leave a + // 2-byte incomplete sequence; ToValidUTF8 must strip it. + val := strings.Repeat("a", spanEnrichmentMaxDefaultValueLen-2) + "€" // 62 + 3 = 65 bytes + + se := newSpanEnrichment() + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: "utf8-flag", DefaultValue: val}) + + var defaults map[string]string + require.NoError(t, json.Unmarshal([]byte(se.GetSpanTags()["ffe_runtime_defaults"]), &defaults)) + got := defaults["utf8-flag"] + assert.True(t, utf8.ValidString(got), "stored value must be valid UTF-8") + // Incomplete \xe2\x82 tail is dropped, only the 62 ASCII bytes survive. + assert.Equal(t, strings.Repeat("a", spanEnrichmentMaxDefaultValueLen-2), got) +} + +// TestSpanEnrichment_InvalidUTF8_NoTruncation verifies that invalid UTF-8 bytes +// are sanitized even when the value is within the length limit. +func TestSpanEnrichment_InvalidUTF8_NoTruncation(t *testing.T) { + invalidUTF8 := "hello\xff\xfeworld" // \xff and \xfe are not valid UTF-8 + + se := newSpanEnrichment() + se.addEvaluation(&FeatureFlagEvaluation{FlagKey: "bad-utf8-flag", DefaultValue: invalidUTF8}) + + var defaults map[string]string + require.NoError(t, json.Unmarshal([]byte(se.GetSpanTags()["ffe_runtime_defaults"]), &defaults)) + got := defaults["bad-utf8-flag"] + assert.True(t, utf8.ValidString(got), "stored value must be valid UTF-8") + assert.Equal(t, "helloworld", got) +} + +func decodeSerialIDs(t *testing.T, enc string) []uint32 { + t.Helper() + + b, err := base64.StdEncoding.DecodeString(enc) + require.NoError(t, err) + + ids := make([]uint32, 0) + var id uint32 + var shift uint + var diff uint32 + for _, v := range b { + diff |= uint32(v&0x7f) << shift + if v&0x80 != 0 { + shift += 7 + continue + } + id += diff + ids = append(ids, id) + diff = 0 + shift = 0 + } + return ids +} diff --git a/openfeature/doc.go b/openfeature/doc.go index 040d5738953..46fb355f8e8 100644 --- a/openfeature/doc.go +++ b/openfeature/doc.go @@ -223,9 +223,16 @@ // (https://github.com/open-feature/go-sdk/tree/main/openfeature/multi) // to implement local overrides during development or testing. // +// - DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED: When set to +// "true", enables span enrichment — feature flag evaluation details are +// recorded as tags on the active trace's root span. Requires +// DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED to also be set to "true". +// Default: false. Note: the added span tags may affect APM billing. +// // Example: // // export DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=true +// export DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED=true // // Standard Datadog environment variables also apply: // diff --git a/openfeature/evaluator.go b/openfeature/evaluator.go index 58021cc3f82..c41a0006fa9 100644 --- a/openfeature/evaluator.go +++ b/openfeature/evaluator.go @@ -34,6 +34,15 @@ type evaluationResult struct { Metadata map[string]any } +const ( + metadataAllocationKey = "dd.allocation.key" + metadataDoLogKey = "dd.doLog" + metadataSerialIDKey = "dd.serialId" + // metadataEvalTimeKey carries the evaluation timestamp (UnixMilli, int64). It is stamped in + // DatadogProvider.evaluate at evaluation entry so EVP first/last bounds use eval-time. + metadataEvalTimeKey = "dd.eval.timestamp_ms" +) + // evaluateFlag evaluates a feature flag with the given context. The caller supplies the // evaluation time (now) so a single timestamp is shared between the allocation time-window // checks here and the EVP eval-time metadata stamped by DatadogProvider.evaluate — avoiding a @@ -92,6 +101,10 @@ func evaluateFlag(flag *flag, defaultValue any, context map[string]any, now time } metadata[metadataDoLogKey] = doLog + if split.SerialID != nil { + metadata[metadataSerialIDKey] = *split.SerialID + } + // Determine reason: // rules matched → TARGETING_MATCH // no rules, shards used → SPLIT diff --git a/openfeature/exposure_hook.go b/openfeature/exposure_hook.go index 31ad834e39a..9282b96df58 100644 --- a/openfeature/exposure_hook.go +++ b/openfeature/exposure_hook.go @@ -15,15 +15,6 @@ import ( "github.com/DataDog/dd-trace-go/v2/internal/log" ) -const ( - // Metadata keys for exposure tracking - metadataAllocationKey = "dd.allocation.key" - metadataDoLogKey = "dd.doLog" - // metadataEvalTimeKey carries the evaluation timestamp (UnixMilli, int64). It is stamped in - // DatadogProvider.evaluate at evaluation entry so EVP first/last bounds use eval-time. - metadataEvalTimeKey = "dd.eval.timestamp_ms" -) - // exposureHook implements the OpenFeature Hook interface to track feature flag exposures. // It captures evaluation details and sends them to the exposure writer for reporting // to Datadog's event platform intake. diff --git a/openfeature/provider.go b/openfeature/provider.go index 188be24a483..ab052d7069c 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -33,7 +33,8 @@ var ( const ( // ffeProductEnvVar is the environment variable to enable the experimental flagging provider - ffeProductEnvVar = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED" + ffeProductEnvVar = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED" + spanEnrichmentEnvVar = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED" // flagEvalCountsEnabledEnvVar is the operator killswitch for the EVP flagevaluation emission path. // Default: true (EVP path is ON by default). Set to "false" to disable only the EVP path // while leaving the OTel feature_flag.evaluations path unaffected. @@ -65,9 +66,10 @@ type DatadogProvider struct { configChange sync.Cond + hooks []openfeature.Hook + // Exposure tracking exposureWriter *exposureWriter - exposureHook *exposureHook // Flag evaluation metrics hook (OTel counter via Finally hook) flagEvalMetricsHook *flagEvalMetricsHook @@ -124,17 +126,40 @@ func newDatadogProvider(config ProviderConfig) *DatadogProvider { evalLoggingHook = newFlagEvalLoggingHook(evalWriter) } + var spanEnrichmentHook *spanEnrichmentHook + if internal.BoolEnv(spanEnrichmentEnvVar, false) { + spanEnrichmentHook = newSpanEnrichmentHook() + log.Debug("openfeature: span enrichment is enabled") + } else { + log.Debug("openfeature: span enrichment is disabled") + } + + hooks := make([]openfeature.Hook, 0, 4) + if exposureLoggingHook != nil { + hooks = append(hooks, exposureLoggingHook) + } + if evalMetricsHook != nil { + hooks = append(hooks, evalMetricsHook) + } + if evalLoggingHook != nil { + hooks = append(hooks, evalLoggingHook) + } + if spanEnrichmentHook != nil { + hooks = append(hooks, spanEnrichmentHook) + } + p := &DatadogProvider{ metadata: openfeature.Metadata{ Name: "Datadog Remote Config Provider", }, + hooks: hooks, exposureWriter: writer, - exposureHook: exposureLoggingHook, flagEvalMetricsHook: evalMetricsHook, flagEvalLoggingWriter: evalWriter, flagEvalLoggingHook: evalLoggingHook, } p.configChange.L = &p.mu + return p } @@ -447,23 +472,10 @@ func (p *DatadogProvider) ObjectEvaluation( } } -// Hooks returns the hooks for this provider. -// This includes the exposure tracking hook, the OTel flag evaluation metrics hook (always), -// and the EVP flagevaluation hook (only when DD_FLAGGING_EVALUATION_COUNTS_ENABLED is true). +// Hooks returns the provider's hooks, built once during Init. +// Returns p.hooks directly to avoid per-evaluation allocations. func (p *DatadogProvider) Hooks() []openfeature.Hook { - var hooks []openfeature.Hook - if p.exposureHook != nil { - hooks = append(hooks, p.exposureHook) - } - // OTel hook is always registered — untouched by the EVP killswitch. - if p.flagEvalMetricsHook != nil { - hooks = append(hooks, p.flagEvalMetricsHook) - } - // EVP hook is nil when the killswitch disabled it. - if p.flagEvalLoggingHook != nil { - hooks = append(hooks, p.flagEvalLoggingHook) - } - return hooks + return p.hooks } // evaluate is the core evaluation method that all type-specific methods use. diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 0bc2b1c1b10..1e3fedc964a 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -356,15 +356,27 @@ var flagEvalBenchProfiles = []flagEvalBenchProfile{ {"scale/2500flags_500users_20fields", 2500, 500, 20}, } +// removeHookType removes all hooks of type T from p.hooks in-place. Used by benchmarks to +// select which hook tiers are active without rebuilding the provider from scratch. +func removeHookType[T openfeature.Hook](p *DatadogProvider) { + out := p.hooks[:0] + for _, h := range p.hooks { + if _, ok := h.(T); !ok { + out = append(out, h) + } + } + p.hooks = out +} + // runFlagEvalBenchmark drives evaluations through a real OpenFeature client so the provider's // registered hooks actually run - calling provider.BooleanEvaluation directly would bypass // Hooks() and every column would measure the same bare evaluator. Flag and targeting keys // rotate to exercise flag- and user-cardinality. // -// configureHooks nils whichever provider hooks the tier under test should exclude (it runs -// before registration; Init does not recreate hooks, so the choice sticks). The 24h flush -// interval keeps the EVP writer from flushing during the run, so no HTTP round-trip enters -// the hot path - isolating hook + aggregator cost. +// configureHooks adjusts p.hooks (via removeHookType) and any associated writer fields for the +// tier under test (runs before registration; Init does not recreate hooks, so the choice sticks). +// The 24h flush interval keeps the EVP writer from flushing during the run, so no HTTP +// round-trip enters the hot path - isolating hook + aggregator cost. func runFlagEvalBenchmark(b *testing.B, configureHooks func(p *DatadogProvider)) { b.Helper() for _, p := range flagEvalBenchProfiles { @@ -409,7 +421,9 @@ func runFlagEvalBenchmark(b *testing.B, configureHooks func(p *DatadogProvider)) // -benchmem -count=3 -cpu=8 func BenchmarkFlagEvaluationNoop(b *testing.B) { runFlagEvalBenchmark(b, func(p *DatadogProvider) { - p.exposureHook = nil + removeHookType[*exposureHook](p) + removeHookType[*flagEvalMetricsHook](p) + removeHookType[*flagEvalLoggingHook](p) p.flagEvalMetricsHook = nil p.flagEvalLoggingHook = nil p.flagEvalLoggingWriter = nil @@ -421,7 +435,8 @@ func BenchmarkFlagEvaluationNoop(b *testing.B) { // are disabled so this column isolates the OTel hook's cost. func BenchmarkFlagEvaluationOTelOnly(b *testing.B) { runFlagEvalBenchmark(b, func(p *DatadogProvider) { - p.exposureHook = nil + removeHookType[*exposureHook](p) + removeHookType[*flagEvalLoggingHook](p) p.flagEvalLoggingHook = nil p.flagEvalLoggingWriter = nil }) @@ -432,7 +447,7 @@ func BenchmarkFlagEvaluationOTelOnly(b *testing.B) { // hook is disabled; the OTel hook, EVP hook, and aggregator all run. func BenchmarkFlagEvaluationOTelPlusEVP(b *testing.B) { runFlagEvalBenchmark(b, func(p *DatadogProvider) { - p.exposureHook = nil + removeHookType[*exposureHook](p) }) } @@ -506,8 +521,7 @@ func BenchmarkFlagEvaluationOTelPlusEVPParallel(b *testing.B) { provider := newDatadogProvider(ProviderConfig{FlagEvaluationFlushInterval: 24 * time.Hour}) // Disable only the exposure hook; OTel hook + EVP hook + aggregator all run, so this // measures the combined Path A + Path B cost under contention. - provider.exposureHook = nil - + removeHookType[*exposureHook](provider) config := makeBenchmarkConfig(p.numFlags) provider.updateConfiguration(config) diff --git a/openfeature/span_enrichment.go b/openfeature/span_enrichment.go new file mode 100644 index 00000000000..3d6f6ece0f7 --- /dev/null +++ b/openfeature/span_enrichment.go @@ -0,0 +1,76 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package openfeature + +import ( + "context" + _ "unsafe" // for go:linkname + + of "github.com/open-feature/go-sdk/openfeature" + + "github.com/DataDog/dd-trace-go/v2/ddtrace/tracer" + "github.com/DataDog/dd-trace-go/v2/internal/log" + iof "github.com/DataDog/dd-trace-go/v2/internal/openfeature" +) + +type spanEnrichmentHook struct { + of.UnimplementedHook +} + +// check that we implement hook interface +var _ of.Hook = (*spanEnrichmentHook)(nil) + +//go:linkname recordFFEEvaluation github.com/DataDog/dd-trace-go/v2/ddtrace/tracer.recordFFEEvaluation +func recordFFEEvaluation(s *tracer.Span, eval *iof.FeatureFlagEvaluation) + +func newSpanEnrichmentHook() *spanEnrichmentHook { + return &spanEnrichmentHook{} +} + +func (h *spanEnrichmentHook) Finally(ctx context.Context, hookContext of.HookContext, evalDetails of.InterfaceEvaluationDetails, hints of.HookHints) { + span, ok := tracer.SpanFromContext(ctx) + if span == nil || !ok { + return + } + + root := span.Root() + if root == nil { + return + } + + eval := buildEvaluation(hookContext.EvaluationContext(), evalDetails) + if eval == nil { + return + } + + recordFFEEvaluation(root, eval) +} + +// buildEvaluation extracts a FeatureFlagEvaluation from OpenFeature hook data. +func buildEvaluation(evalCtx of.EvaluationContext, evalDetails of.InterfaceEvaluationDetails) *iof.FeatureFlagEvaluation { + if raw, ok := evalDetails.FlagMetadata[metadataSerialIDKey]; ok { + sid, ok := raw.(uint32) + if !ok { + log.Debug("openfeature: span enrichment: malformed serial id in metadata") + return nil + } + eval := &iof.FeatureFlagEvaluation{ + FlagKey: evalDetails.FlagKey, + SerialID: &sid, + } + if doLog, err := evalDetails.FlagMetadata.GetBool(metadataDoLogKey); err == nil && doLog { + eval.Subject = evalCtx.TargetingKey() + } + return eval + } else if evalDetails.Variant == "" { + // Runtime default was used. + return &iof.FeatureFlagEvaluation{ + FlagKey: evalDetails.FlagKey, + DefaultValue: evalDetails.Value, + } + } + return nil +} diff --git a/openfeature/span_enrichment_test.go b/openfeature/span_enrichment_test.go new file mode 100644 index 00000000000..5d9c805217f --- /dev/null +++ b/openfeature/span_enrichment_test.go @@ -0,0 +1,373 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package openfeature + +import ( + "context" + "encoding/base64" + "encoding/json" + "testing" + + rc "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" + of "github.com/open-feature/go-sdk/openfeature" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/dd-trace-go/v2/ddtrace/mocktracer" + "github.com/DataDog/dd-trace-go/v2/ddtrace/tracer" + i "github.com/DataDog/dd-trace-go/v2/internal/openfeature" +) + +func TestBuildEvaluation(t *testing.T) { + tests := []struct { + name string + evalCtx of.EvaluationContext + details of.InterfaceEvaluationDetails + expected *i.FeatureFlagEvaluation + }{ + { + name: "experiment with subject", + evalCtx: of.NewEvaluationContext("user-123", map[string]any{}), + details: of.InterfaceEvaluationDetails{ + Value: true, + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "experiment-flag", + ResolutionDetail: of.ResolutionDetail{ + Variant: "v1", + FlagMetadata: of.FlagMetadata{ + metadataSerialIDKey: uint32(42), + metadataDoLogKey: true, + }, + }, + }, + }, + expected: &i.FeatureFlagEvaluation{ + FlagKey: "experiment-flag", + SerialID: uint32Ptr(42), + Subject: "user-123", + }, + }, + { + name: "experiment without subject logging", + evalCtx: of.NewEvaluationContext("user-456", map[string]any{}), + details: of.InterfaceEvaluationDetails{ + Value: true, + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "no-log-flag", + ResolutionDetail: of.ResolutionDetail{ + Variant: "v2", + FlagMetadata: of.FlagMetadata{ + metadataSerialIDKey: uint32(101), + metadataDoLogKey: false, + }, + }, + }, + }, + expected: &i.FeatureFlagEvaluation{ + FlagKey: "no-log-flag", + SerialID: uint32Ptr(101), + }, + }, + { + name: "experiment without doLog metadata", + evalCtx: of.NewEvaluationContext("user-789", map[string]any{}), + details: of.InterfaceEvaluationDetails{ + Value: true, + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "missing-do-log-flag", + ResolutionDetail: of.ResolutionDetail{ + Variant: "v3", + FlagMetadata: of.FlagMetadata{ + metadataSerialIDKey: uint32(7), + }, + }, + }, + }, + expected: &i.FeatureFlagEvaluation{ + FlagKey: "missing-do-log-flag", + SerialID: uint32Ptr(7), + }, + }, + { + name: "malformed serial id", + evalCtx: of.NewEvaluationContext("user-000", map[string]any{}), + details: of.InterfaceEvaluationDetails{ + Value: true, + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "bad-flag", + ResolutionDetail: of.ResolutionDetail{ + Variant: "v1", + FlagMetadata: of.FlagMetadata{ + metadataSerialIDKey: "42", + metadataDoLogKey: true, + }, + }, + }, + }, + expected: nil, + }, + { + name: "runtime default", + evalCtx: of.NewEvaluationContext("", map[string]any{}), + details: of.InterfaceEvaluationDetails{ + Value: "default-val", + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "default-flag", + ResolutionDetail: of.ResolutionDetail{ + Variant: "", + }, + }, + }, + expected: &i.FeatureFlagEvaluation{ + FlagKey: "default-flag", + DefaultValue: "default-val", + }, + }, + { + name: "non-default without serial id", + evalCtx: of.NewEvaluationContext("user-999", map[string]any{}), + details: of.InterfaceEvaluationDetails{ + Value: true, + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "plain-flag", + ResolutionDetail: of.ResolutionDetail{ + Variant: "v1", + }, + }, + }, + expected: nil, + }, + { + // When an error occurs the SDK sets ErrorCode and leaves Variant empty. + // buildEvaluation sees Variant=="" and records the default value that + // was returned to the caller, matching runtime-default behaviour. + name: "error evaluation with empty variant", + evalCtx: of.NewEvaluationContext("user-111", map[string]any{}), + details: of.InterfaceEvaluationDetails{ + Value: "fallback", + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "error-flag", + ResolutionDetail: of.ResolutionDetail{ + Variant: "", + ErrorCode: of.GeneralCode, + }, + }, + }, + expected: &i.FeatureFlagEvaluation{ + FlagKey: "error-flag", + DefaultValue: "fallback", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildEvaluation(tt.evalCtx, tt.details) + if tt.expected == nil { + assert.Nil(t, got) + return + } + + require.NotNil(t, got) + assert.Equal(t, tt.expected.FlagKey, got.FlagKey) + assert.Equal(t, tt.expected.Subject, got.Subject) + assert.Equal(t, tt.expected.DefaultValue, got.DefaultValue) + if tt.expected.SerialID == nil { + assert.Nil(t, got.SerialID) + return + } + require.NotNil(t, got.SerialID) + assert.Equal(t, *tt.expected.SerialID, *got.SerialID) + }) + } +} + +func TestSpanEnrichment_Integration(t *testing.T) { + t.Setenv(spanEnrichmentEnvVar, "true") + + mt := mocktracer.Start() + defer mt.Stop() + + provider := newDatadogProvider(ProviderConfig{}) + status := processConfigUpdate(provider, "datadog/2/ASM_FEATURES/test/config", []byte(`{ + "createdAt":"2026-01-01T00:00:00Z", + "format":"SERVER", + "environment":{"name":"test"}, + "flags":{ + "experiment-flag":{ + "key":"experiment-flag", + "enabled":true, + "variationType":"BOOLEAN", + "variations":{ + "on":{"key":"on","value":true}, + "off":{"key":"off","value":false} + }, + "allocations":[{ + "key":"us-rollout", + "doLog":true, + "rules":[{"conditions":[{"operator":"ONE_OF","attribute":"country","value":["US"]}]}], + "splits":[{"variationKey":"on","serialId":42,"shards":[]}] + }] + }, + "default-flag":{ + "key":"default-flag", + "enabled":true, + "variationType":"STRING", + "variations":{"configured":{"key":"configured","value":"configured-val"}}, + "allocations":[{ + "key":"uk-rollout", + "doLog":true, + "rules":[{"conditions":[{"operator":"ONE_OF","attribute":"country","value":["UK"]}]}], + "splits":[{"variationKey":"configured","serialId":101,"shards":[]}] + }] + } + } + }`)) + require.Equal(t, rc.ApplyStateAcknowledged, status.State) + + domain := "span-enrichment-test-app" + require.NoError(t, of.SetNamedProviderAndWait(domain, provider)) + client := of.NewClient(domain) + + rootSpan := tracer.StartSpan("http.request") + rootCtx := tracer.ContextWithSpan(context.Background(), rootSpan) + childSpan := tracer.StartSpan("db.query", tracer.ChildOf(rootSpan.Context())) + childCtx := tracer.ContextWithSpan(rootCtx, childSpan) + + evalCtx := of.NewEvaluationContext("user-123", map[string]any{"country": "US"}) + gotBool, err := client.BooleanValue(childCtx, "experiment-flag", false, evalCtx) + require.NoError(t, err) + assert.True(t, gotBool) + + gotString, err := client.StringValue(childCtx, "default-flag", "default-val", evalCtx) + require.NoError(t, err) + assert.Equal(t, "default-val", gotString) + + childSpan.Finish() + rootSpan.Finish() + + finished := mt.FinishedSpans() + require.Len(t, finished, 2) + + var rootMock, childMock *mocktracer.Span + for _, s := range finished { + if s.OperationName() == "http.request" { + rootMock = s + } else if s.OperationName() == "db.query" { + childMock = s + } + } + require.NotNil(t, rootMock) + require.NotNil(t, childMock) + + for k := range childMock.Tags() { + assert.NotContains(t, k, "ffe_") + } + + tags := rootMock.Tags() + assert.Contains(t, tags, "ffe_flags_enc") + assert.Contains(t, tags, "ffe_subjects_enc") + assert.Contains(t, tags, "ffe_runtime_defaults") + + flagsEnc, ok := tags["ffe_flags_enc"].(string) + require.True(t, ok) + flagsDec, err := base64.StdEncoding.DecodeString(flagsEnc) + require.NoError(t, err) + assert.Equal(t, []byte{42}, flagsDec) + + subjectsEnc, ok := tags["ffe_subjects_enc"].(string) + require.True(t, ok) + var subjects map[string]string + require.NoError(t, json.Unmarshal([]byte(subjectsEnc), &subjects)) + expectedUser123Enc := base64.StdEncoding.EncodeToString([]byte{42}) + user123Hash := "fcdec6df4d44dbc637c7c5b58efface52a7f8a88535423430255be0bb89bedd8" + assert.Equal(t, map[string]string{user123Hash: expectedUser123Enc}, subjects) + + defaultsEnc, ok := tags["ffe_runtime_defaults"].(string) + require.True(t, ok) + var defaults map[string]string + require.NoError(t, json.Unmarshal([]byte(defaultsEnc), &defaults)) + assert.Equal(t, map[string]string{"default-flag": "default-val"}, defaults) +} + +// setupEnrichmentProvider creates a minimal provider with a single boolean flag +// assigned to a US-only allocation (serialId=42, doLog=true) and registers it +// under the given OpenFeature domain name. +func setupEnrichmentProvider(t *testing.T, domain string) *of.Client { + t.Helper() + provider := newDatadogProvider(ProviderConfig{}) + status := processConfigUpdate(provider, "datadog/2/ASM_FEATURES/test/config", []byte(`{ + "createdAt":"2026-01-01T00:00:00Z", + "format":"SERVER", + "environment":{"name":"test"}, + "flags":{ + "experiment-flag":{ + "key":"experiment-flag", + "enabled":true, + "variationType":"BOOLEAN", + "variations":{ + "on":{"key":"on","value":true}, + "off":{"key":"off","value":false} + }, + "allocations":[{ + "key":"us-rollout", + "doLog":true, + "rules":[{"conditions":[{"operator":"ONE_OF","attribute":"country","value":["US"]}]}], + "splits":[{"variationKey":"on","serialId":42,"shards":[]}] + }] + } + } + }`)) + require.Equal(t, rc.ApplyStateAcknowledged, status.State) + require.NoError(t, of.SetNamedProviderAndWait(domain, provider)) + return of.NewClient(domain) +} + +func TestSpanEnrichment_NoSpanInContext(t *testing.T) { + t.Setenv(spanEnrichmentEnvVar, "true") + + mt := mocktracer.Start() + defer mt.Stop() + + client := setupEnrichmentProvider(t, "span-enrichment-no-span") + evalCtx := of.NewEvaluationContext("user-123", map[string]any{"country": "US"}) + + // context.Background() carries no span; hook must return early without panicking. + got, err := client.BooleanValue(context.Background(), "experiment-flag", false, evalCtx) + require.NoError(t, err) + assert.True(t, got) + + // No spans created, so nothing to check for ffe tags. + assert.Empty(t, mt.FinishedSpans()) +} + +func TestSpanEnrichment_AfterRootFinished(t *testing.T) { + t.Setenv(spanEnrichmentEnvVar, "true") + + mt := mocktracer.Start() + defer mt.Stop() + + client := setupEnrichmentProvider(t, "span-enrichment-after-finished") + evalCtx := of.NewEvaluationContext("user-123", map[string]any{"country": "US"}) + + rootSpan := tracer.StartSpan("http.request") + rootSpan.Finish() + rootCtx := tracer.ContextWithSpan(context.Background(), rootSpan) + + // Evaluate after the root span is already finished – must not panic. + got, err := client.BooleanValue(rootCtx, "experiment-flag", false, evalCtx) + require.NoError(t, err) + assert.True(t, got) + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + // ffe tags must not appear on an already-finished span. + for k := range finished[0].Tags() { + assert.NotContains(t, k, "ffe_") + } +} + +func uint32Ptr(v uint32) *uint32 { return &v } diff --git a/openfeature/types.go b/openfeature/types.go index 5be6708966e..277ddf06602 100644 --- a/openfeature/types.go +++ b/openfeature/types.go @@ -147,9 +147,8 @@ type split struct { // All shards must match for the split to apply (AND logic) Shards []*shard `json:"shards"` // VariationKey is the key of the variation to return if this split matches - VariationKey string `json:"variationKey"` - // ExtraLogging contains additional metadata for logging purposes - ExtraLogging map[string]string `json:"extraLogging,omitempty"` + VariationKey string `json:"variationKey"` + SerialID *uint32 `json:"serialId,omitempty"` } // shard defines a portion of traffic using consistent hashing.