From 0d3a4d2a48b5a32276a40930ba569a55569c5833 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 11 Jun 2026 21:59:10 -0400 Subject: [PATCH 01/37] test(01-01): add failing test stubs + signature-only production stubs for EVP flagevaluation - flagevaluation.go: signature-only stubs for aggregator, writer, pruneContext, extractEvalDetails, types (evaluationAggregationKey, evaluationDegradedKey, evaluationUltraDegradedKey, evaluationEntry, flagEvaluationEvent, flagEvaluationPayload, flagEvalDDContext, flagEvaluationWriter, evalDetails), and constants (flagEvaluationEndpoint, maxContextFields, maxFieldLength, defaultEvalGlobalCap, defaultEvalPerFlagCap, defaultEvalDegradedCap, defaultFlagEvalFlushInterval); all methods panic("not implemented") - flagevaluation_hook.go: signature-only stubs for flagEvaluationHook.Finally and isRuntimeDefault; panics with "not implemented" - flagevaluation_test.go: failing RED tests: TestPruneContext, TestFlagEvaluationPayloadSchema, TestAggregatorCollisionSafety, TestAggregatorConcurrentMinMax, TestAggregatorCapOverflow - flagevaluation_hook_test.go: failing RED tests: TestIsRuntimeDefault, TestFlagEvaluationHookFinally, TestFlagEvaluationHookContextCancelled - flageval_metrics.go untouched (PRES-01 invariant) - Tests fail on panic("not implemented"), not compile errors (clean RED state) --- openfeature/flagevaluation.go | 222 +++++++++++ openfeature/flagevaluation_hook.go | 48 +++ openfeature/flagevaluation_hook_test.go | 240 ++++++++++++ openfeature/flagevaluation_test.go | 481 ++++++++++++++++++++++++ 4 files changed, 991 insertions(+) create mode 100644 openfeature/flagevaluation.go create mode 100644 openfeature/flagevaluation_hook.go create mode 100644 openfeature/flagevaluation_hook_test.go create mode 100644 openfeature/flagevaluation_test.go diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go new file mode 100644 index 00000000000..b85465f981c --- /dev/null +++ b/openfeature/flagevaluation.go @@ -0,0 +1,222 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +// Package openfeature provides flag evaluation EVP emission. +// This file contains signature-only stubs; the real implementation is in plan 02. +package openfeature + +import ( + "net/http" + "net/url" + "sync" + "time" + + jsoniter "github.com/json-iterator/go" + of "github.com/open-feature/go-sdk/openfeature" +) + +const ( + // defaultFlagEvalFlushInterval is the flush interval for EVP flag evaluation events. + // D-05: dedicated 10 s timer (contract value); separate from exposureWriter's 1 s interval. + defaultFlagEvalFlushInterval = 10 * time.Second + + // flagEvaluationEndpoint is the EVP proxy endpoint for flag evaluation events. + flagEvaluationEndpoint = "/evp_proxy/v2/api/v2/flagevaluations" + + // Context pruning limits (D-07) — mirror worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH. + maxContextFields = 256 + maxFieldLength = 256 + + // Aggregation caps (D-08/D-09/CONT-10). + defaultEvalGlobalCap = 65_536 // bounds total distinct buckets across all tiers + defaultEvalPerFlagCap = 10_000 // bounds full-fidelity buckets per flag + defaultEvalDegradedCap = 10_000 // bounds degraded map (absent from POC — Gate 8 fix) +) + +// evaluationAggregationKey is a struct-keyed map key (collision-free for enumerable dims). +// contextHash is a uint64 FNV-1a hash of the pruned context — NOT the sole map key. +// Replaces the FNV-1a-keyed map from PR #4874 POC (CONT-05 / source_comment_id 3395004724). +type evaluationAggregationKey struct { + flagKey string + variant string + allocationKey string + targetingRuleKey string + reason string + targetingKey string + contextHash uint64 // context is map[string]any (not comparable); hash is a discriminator only +} + +// evaluationDegradedKey is the key for the degraded aggregation map. +// Drops targeting key, context, and targeting rule key relative to the full key. +type evaluationDegradedKey struct { + flagKey string + variant string + allocationKey string + reason string +} + +// evaluationUltraDegradedKey is the key for the ultra-degraded aggregation map. +// Contains only flag key + variant; bounded by flag×variant enumeration. +type evaluationUltraDegradedKey struct { + flagKey string + variant string +} + +// evaluationEntry holds per-window counts and time bounds for one aggregation bucket. +type evaluationEntry struct { + count int64 + firstEvaluation int64 // milliseconds — min across all recordings in this window + lastEvaluation int64 // milliseconds — max across all recordings in this window + runtimeDefault bool + // For full tier only: + targetingKey string + contextAttrs map[string]any + errorMessage string +} + +// flagEvaluationAggregator holds the three-tier aggregation maps. +type flagEvaluationAggregator struct { + mu sync.Mutex + full map[evaluationAggregationKey]*evaluationEntry + degraded map[evaluationDegradedKey]*evaluationEntry + ultraDeg map[evaluationUltraDegradedKey]*evaluationEntry + perFlagFull map[string]int // flagKey → count of full-fidelity entries for this flag + globalCount int + globalCap int + perFlagCap int + degradedCap int +} + +// flagEvaluationEvent matches flagevaluation.json — required fields always present; +// optional fields use omitempty (absent = schema-valid for degraded/ultra-degraded tiers). +type flagEvaluationEvent struct { + Timestamp int64 `json:"timestamp"` + Flag flagEvalFlag `json:"flag"` + FirstEvaluation int64 `json:"first_evaluation"` + LastEvaluation int64 `json:"last_evaluation"` + EvaluationCount int64 `json:"evaluation_count"` + RuntimeDefault bool `json:"runtime_default_used,omitempty"` + TargetingKey string `json:"targeting_key,omitempty"` + Variant *flagEvalVariant `json:"variant,omitempty"` + Allocation *flagEvalAllocation `json:"allocation,omitempty"` + TargetingRule *flagEvalTargetingRule `json:"targeting_rule,omitempty"` + Error *flagEvalError `json:"error,omitempty"` + Context *flagEvalEventContext `json:"context,omitempty"` +} + +// flagEvalFlag holds the flag key. +type flagEvalFlag struct{ Key string `json:"key"` } + +// flagEvalVariant holds the variant key. +type flagEvalVariant struct{ Key string `json:"key"` } + +// flagEvalAllocation holds the allocation key. +type flagEvalAllocation struct{ Key string `json:"key"` } + +// flagEvalTargetingRule holds the targeting rule key. +type flagEvalTargetingRule struct{ Key string `json:"key"` } + +// flagEvalError holds the error message. +type flagEvalError struct{ Message string `json:"message"` } + +// flagEvalEventContext holds the per-event context object. +type flagEvalEventContext struct { + Evaluation map[string]any `json:"evaluation,omitempty"` + DD *flagEvalContextDD `json:"dd,omitempty"` +} + +// flagEvalContextDD holds the Datadog-specific context inside per-event context.dd. +type flagEvalContextDD struct { + Service string `json:"service,omitempty"` +} + +// flagEvaluationPayload matches batchedflagevaluations.json. +type flagEvaluationPayload struct { + Context flagEvalDDContext `json:"context"` + FlagEvaluations []flagEvaluationEvent `json:"flagEvaluations"` +} + +// flagEvalDDContext carries service/env/version for the batch-level context. +type flagEvalDDContext struct { + Service string `json:"service"` + Env string `json:"env,omitempty"` + Version string `json:"version,omitempty"` +} + +// flagEvaluationWriter manages aggregation and periodic flushing of EVP flag evaluation events. +type flagEvaluationWriter struct { + aggregator flagEvaluationAggregator + flushInterval time.Duration + httpClient *http.Client + agentURL *url.URL + ddContext flagEvalDDContext // service/env/version — same source as exposureContext + ticker *time.Ticker + stopChan chan struct{} + stopped bool + jsonConfig jsoniter.API +} + +// evalDetails holds extracted flag evaluation fields for EVP aggregation. +// Used only by flagEvaluationHook; does NOT replace extraction in flageval_metrics.go. +type evalDetails struct { + flagKey string + variant string + reason string + allocationKey string + targetingRuleKey string + targetingKey string + errorMessage string + runtimeDefault bool +} + +// newFlagEvaluationWriter creates a new flag evaluation writer. +// Plan 02 implements the body; this is a signature-only stub. +func newFlagEvaluationWriter(_ ProviderConfig) *flagEvaluationWriter { + panic("not implemented") +} + +// start begins the periodic flushing — called from InitWithContext(), NOT from the constructor. +// Plan 02 implements the body. +func (w *flagEvaluationWriter) start() { + panic("not implemented") +} + +// stop stops the flush ticker and marks the writer as stopped. +// Plan 02 implements the body. +func (w *flagEvaluationWriter) stop() { + panic("not implemented") +} + +// flush sends aggregated evaluation events to the Datadog Agent EVP proxy. +// Plan 02 implements the body. +func (w *flagEvaluationWriter) flush() { + panic("not implemented") +} + +// record adds an evaluation to the aggregation buffer. +// Plan 02 implements the body. +func (w *flagEvaluationWriter) record(_ of.HookContext, _ of.InterfaceEvaluationDetails) { + panic("not implemented") +} + +// add adds one evaluation observation to the aggregator. +// Plan 02 implements the body. +func (a *flagEvaluationAggregator) add(_ evalDetails, _ map[string]any, _ int64) { + panic("not implemented") +} + +// pruneContext applies 256-field / 256-char limits before buffering (D-07). +// Mirrors worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH exactly. +// Plan 02 implements the body. +func pruneContext(_ map[string]any) map[string]any { + panic("not implemented") +} + +// extractEvalDetails extracts EVP-relevant fields from hook context and evaluation details. +// Used only by flagEvaluationHook (flageval_metrics.go is NOT modified — D-06/PRES-01). +// Plan 02 implements the body. +func extractEvalDetails(_ of.HookContext, _ of.InterfaceEvaluationDetails) evalDetails { + panic("not implemented") +} diff --git a/openfeature/flagevaluation_hook.go b/openfeature/flagevaluation_hook.go new file mode 100644 index 00000000000..87695e7fbad --- /dev/null +++ b/openfeature/flagevaluation_hook.go @@ -0,0 +1,48 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +// Package openfeature provides the EVP flag evaluation OpenFeature hook. +// This file contains signature-only stubs; the real implementation is in plan 02. +package openfeature + +import ( + "context" + + of "github.com/open-feature/go-sdk/openfeature" +) + +// flagEvaluationHook implements the OpenFeature Hook interface to record EVP flagevaluation events. +// It uses the Finally hook stage (same as flagEvalHook) to cover success, error, and default paths. +// This satisfies CONT-09: Finally fires on error/default paths, unlike After (PR #4874 defect). +type flagEvaluationHook struct { + of.UnimplementedHook + writer *flagEvaluationWriter +} + +// newFlagEvaluationHook creates a new EVP flag evaluation hook. +func newFlagEvaluationHook(w *flagEvaluationWriter) *flagEvaluationHook { + return &flagEvaluationHook{writer: w} +} + +// Finally is called after every flag evaluation (success or error). +// Using Finally (not After) ensures error-path and provider-not-ready evaluations are counted (CONT-09). +// Plan 02 implements the body; this is a signature-only stub. +func (h *flagEvaluationHook) Finally( + ctx context.Context, + hookContext of.HookContext, + details of.InterfaceEvaluationDetails, + _ of.HookHints, +) { + panic("not implemented") +} + +// isRuntimeDefault returns true when the caller's supplied default value was returned. +// Primary signal: absent variant key (flag-not-found, provider-not-ready, type-mismatch, no allocation). +// Secondary: explicit DEFAULT or DISABLED reason (belt-and-suspenders). +// Satisfies CONT-07 (source_comment_id 3395344504). +// Plan 02 implements the body; this is a signature-only stub. +func isRuntimeDefault(_ of.InterfaceEvaluationDetails) bool { + panic("not implemented") +} diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go new file mode 100644 index 00000000000..ca7244abe46 --- /dev/null +++ b/openfeature/flagevaluation_hook_test.go @@ -0,0 +1,240 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +package openfeature + +import ( + "context" + "testing" + "time" + + jsoniter "github.com/json-iterator/go" + of "github.com/open-feature/go-sdk/openfeature" +) + +// setupTestWriter creates a flagEvaluationWriter configured for unit testing. +// The writer uses a large flush interval (24 h) so no automatic flush fires during tests. +func setupTestWriter(t *testing.T) *flagEvaluationWriter { + t.Helper() + return &flagEvaluationWriter{ + flushInterval: 24 * time.Hour, // effectively disabled; tests control flush manually + jsonConfig: jsoniter.Config{}.Froze(), + stopChan: make(chan struct{}), + aggregator: flagEvaluationAggregator{ + full: make(map[evaluationAggregationKey]*evaluationEntry), + degraded: make(map[evaluationDegradedKey]*evaluationEntry), + ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), + perFlagFull: make(map[string]int), + globalCap: 10, + perFlagCap: 3, + degradedCap: 3, + }, + } +} + +// makeHookContext creates an of.HookContext for testing. +func makeHookContext(flagKey string, targetingKey string, attrs map[string]any) of.HookContext { + evalCtx := of.NewEvaluationContext(targetingKey, attrs) + return of.NewHookContext( + flagKey, + of.Boolean, + false, + of.ClientMetadata{}, + of.Metadata{}, + evalCtx, + ) +} + +// makeEvalDetails constructs an InterfaceEvaluationDetails for hook testing. +func makeEvalDetails(variant string, reason of.Reason, errorCode of.ErrorCode, metadata ...of.FlagMetadata) of.InterfaceEvaluationDetails { + d := of.InterfaceEvaluationDetails{ + EvaluationDetails: of.EvaluationDetails{ + ResolutionDetail: of.ResolutionDetail{ + Variant: variant, + Reason: reason, + ErrorCode: errorCode, + }, + }, + } + if len(metadata) > 0 { + d.FlagMetadata = metadata[0] + } + return d +} + +// TestIsRuntimeDefault verifies the runtime-default detection rule (CONT-07 / source_comment_id 3395344504). +// Primary signal: variant=="" → true. +// Secondary signal: reason==DEFAULT or reason==DISABLED → true even if variant is non-empty. +// +// It must fail RED: isRuntimeDefault panics with "not implemented". +func TestIsRuntimeDefault(t *testing.T) { + tests := []struct { + name string + details of.InterfaceEvaluationDetails + want bool + }{ + { + name: "empty variant is runtime default", + details: makeEvalDetails("", of.TargetingMatchReason, ""), + want: true, + }, + { + name: "reason DEFAULT is runtime default", + details: makeEvalDetails("", of.DefaultReason, ""), + want: true, + }, + { + name: "reason DISABLED is runtime default", + details: makeEvalDetails("", of.DisabledReason, ""), + want: true, + }, + { + name: "variant present with TARGETING_MATCH is NOT runtime default", + details: makeEvalDetails("on", of.TargetingMatchReason, ""), + want: false, + }, + { + name: "variant present with SPLIT is NOT runtime default", + details: makeEvalDetails("variant-a", of.SplitReason, ""), + want: false, + }, + { + name: "variant present with STATIC is NOT runtime default", + details: makeEvalDetails("on", of.StaticReason, ""), + want: false, + }, + { + name: "empty variant with ERROR reason is runtime default (primary: variant absent)", + details: makeEvalDetails("", of.ErrorReason, of.FlagNotFoundCode), + want: true, + }, + { + name: "variant present with DEFAULT reason is runtime default (secondary belt-and-suspenders)", + details: makeEvalDetails("default-variant", of.DefaultReason, ""), + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := isRuntimeDefault(tc.details) + if got != tc.want { + t.Errorf("isRuntimeDefault() = %v, want %v", got, tc.want) + } + }) + } +} + +// TestFlagEvaluationHookFinally verifies that the Finally hook records an entry for +// success, error-reason, and provider-not-ready paths (CONT-09 / source_comment_id 3385309423). +// +// It must fail RED: the hook's Finally method panics with "not implemented". +func TestFlagEvaluationHookFinally(t *testing.T) { + t.Run("success path records entry", func(t *testing.T) { + w := setupTestWriter(t) + hook := newFlagEvaluationHook(w) + + hookCtx := makeHookContext("test-flag", "user-123", map[string]any{"country": "US"}) + details := makeEvalDetails("on", of.TargetingMatchReason, "", of.FlagMetadata{ + metadataAllocationKey: "default-alloc", + }) + + hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + + w.aggregator.mu.Lock() + defer w.aggregator.mu.Unlock() + + total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) + if total == 0 { + t.Error("expected Finally to record an entry on success path, got none") + } + }) + + t.Run("error-reason path records entry (flag-not-found)", func(t *testing.T) { + w := setupTestWriter(t) + hook := newFlagEvaluationHook(w) + + hookCtx := makeHookContext("missing-flag", "user-123", nil) + details := makeEvalDetails("", of.ErrorReason, of.FlagNotFoundCode) + + hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + + w.aggregator.mu.Lock() + defer w.aggregator.mu.Unlock() + + total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) + if total == 0 { + t.Error("expected Finally to record an entry on error-reason path, got none") + } + }) + + t.Run("provider-not-ready path records entry", func(t *testing.T) { + w := setupTestWriter(t) + hook := newFlagEvaluationHook(w) + + hookCtx := makeHookContext("any-flag", "", nil) + details := makeEvalDetails("", of.ErrorReason, of.ProviderNotReadyCode) + + hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + + w.aggregator.mu.Lock() + defer w.aggregator.mu.Unlock() + + total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) + if total == 0 { + t.Error("expected Finally to record an entry on provider-not-ready path, got none") + } + }) + + t.Run("DEFAULT reason path records entry with runtime_default_used=true", func(t *testing.T) { + w := setupTestWriter(t) + hook := newFlagEvaluationHook(w) + + hookCtx := makeHookContext("absent-flag", "user-456", nil) + details := makeEvalDetails("", of.DefaultReason, "") + + hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + + w.aggregator.mu.Lock() + defer w.aggregator.mu.Unlock() + + total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) + if total == 0 { + t.Error("expected Finally to record an entry on DEFAULT-reason path, got none") + } + + // The recorded entry should have runtimeDefault=true + for _, e := range w.aggregator.full { + if !e.runtimeDefault { + t.Error("expected runtimeDefault=true for DEFAULT-reason evaluation") + } + } + }) +} + +// TestFlagEvaluationHookContextCancelled verifies that a cancelled context causes +// Finally to return without recording (context safety / no zombie writes). +// +// It must fail RED: the hook's Finally method panics with "not implemented". +func TestFlagEvaluationHookContextCancelled(t *testing.T) { + w := setupTestWriter(t) + hook := newFlagEvaluationHook(w) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel BEFORE calling Finally + + hookCtx := makeHookContext("test-flag", "user-123", nil) + details := makeEvalDetails("on", of.TargetingMatchReason, "") + + hook.Finally(ctx, hookCtx, details, of.HookHints{}) + + w.aggregator.mu.Lock() + defer w.aggregator.mu.Unlock() + + total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) + if total != 0 { + t.Errorf("expected no entries when context is cancelled, got %d", total) + } +} diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go new file mode 100644 index 00000000000..70b8eccb65b --- /dev/null +++ b/openfeature/flagevaluation_test.go @@ -0,0 +1,481 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +package openfeature + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +// setupTestAggregator creates a flagEvaluationAggregator with small caps for testing. +// Caps are deliberately small to trigger tier-cascade behavior in unit tests. +func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { + t.Helper() + return &flagEvaluationAggregator{ + full: make(map[evaluationAggregationKey]*evaluationEntry), + degraded: make(map[evaluationDegradedKey]*evaluationEntry), + ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), + perFlagFull: make(map[string]int), + globalCap: 10, // small cap to trigger overflow in tests + perFlagCap: 3, + degradedCap: 3, + } +} + +// TestPruneContext verifies that pruneContext applies the 256-field / 256-char limits +// before evaluation context enters the aggregation buffer (D-07, CONT-03, T-DoS-mem). +// +// It must fail RED: pruneContext panics with "not implemented". +func TestPruneContext(t *testing.T) { + t.Run("300 fields truncated to exactly 256", func(t *testing.T) { + raw := make(map[string]any, 300) + for i := range 300 { + raw[fmt.Sprintf("key%d", i)] = fmt.Sprintf("value%d", i) + } + + out := pruneContext(raw) + + if len(out) != 256 { + t.Errorf("expected exactly 256 fields after prune, got %d", len(out)) + } + }) + + t.Run("string value exceeding 256 chars is dropped", func(t *testing.T) { + longVal := strings.Repeat("x", 300) // 300 chars > maxFieldLength(256) + raw := map[string]any{ + "short": "ok", + "long": longVal, + } + + out := pruneContext(raw) + + if _, ok := out["long"]; ok { + t.Error("expected long string value to be dropped from pruned context") + } + if _, ok := out["short"]; !ok { + t.Error("expected short string value to be retained in pruned context") + } + }) + + t.Run("nil input returns nil", func(t *testing.T) { + out := pruneContext(nil) + if out != nil { + t.Errorf("expected nil for nil input, got %v", out) + } + }) + + t.Run("empty input returns nil or empty", func(t *testing.T) { + out := pruneContext(map[string]any{}) + if out != nil && len(out) != 0 { + t.Errorf("expected nil or empty for empty input, got %v", out) + } + }) + + t.Run("non-string values are retained regardless of notional length", func(t *testing.T) { + raw := map[string]any{ + "intVal": 42, + "boolVal": true, + } + out := pruneContext(raw) + if out["intVal"] == nil { + t.Error("expected integer value to be retained") + } + if out["boolVal"] == nil { + t.Error("expected boolean value to be retained") + } + }) +} + +// TestFlagEvaluationPayloadSchema verifies that full, degraded, and ultra-degraded events +// marshal to JSON that omits the expected optional fields per tier while always including +// the 5 required fields (CONT-04, CONT-02). +// +// It must fail RED: the aggregator.add method panics with "not implemented". +func TestFlagEvaluationPayloadSchema(t *testing.T) { + nowMs := time.Now().UnixMilli() + + t.Run("full tier has all required fields", func(t *testing.T) { + event := flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "test-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 1, + Variant: &flagEvalVariant{Key: "on"}, + TargetingKey: "user-123", + Context: &flagEvalEventContext{ + Evaluation: map[string]any{"country": "US"}, + }, + } + + b, err := json.Marshal(event) + if err != nil { + t.Fatalf("failed to marshal full event: %v", err) + } + + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + // Required fields must be present + for _, req := range []string{"timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count"} { + if _, ok := m[req]; !ok { + t.Errorf("full tier: required field %q missing from marshaled JSON", req) + } + } + + // flag.key must be present + if flagObj, ok := m["flag"].(map[string]any); !ok { + t.Error("full tier: flag is not an object") + } else if _, ok := flagObj["key"]; !ok { + t.Error("full tier: flag.key missing") + } + }) + + t.Run("degraded tier omits targeting_key and context.evaluation", func(t *testing.T) { + // Degraded tier: no targeting_key, no context.evaluation; variant + allocation present + event := flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "test-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 5, + Variant: &flagEvalVariant{Key: "on"}, + Allocation: &flagEvalAllocation{Key: "default"}, + // TargetingKey: intentionally absent + // Context: intentionally absent + } + + b, err := json.Marshal(event) + if err != nil { + t.Fatalf("failed to marshal degraded event: %v", err) + } + + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + // Required fields must be present + for _, req := range []string{"timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count"} { + if _, ok := m[req]; !ok { + t.Errorf("degraded tier: required field %q missing", req) + } + } + + // targeting_key must be absent + if _, ok := m["targeting_key"]; ok { + t.Error("degraded tier: targeting_key should be absent") + } + + // context must be absent + if _, ok := m["context"]; ok { + t.Error("degraded tier: context should be absent") + } + }) + + t.Run("ultra-degraded tier has only required fields", func(t *testing.T) { + // Ultra-degraded: only flag key + counts; no variant, allocation, targeting, context + event := flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "test-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 1000, + // All optional fields intentionally absent + } + + b, err := json.Marshal(event) + if err != nil { + t.Fatalf("failed to marshal ultra-degraded event: %v", err) + } + + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + // Required fields must be present + for _, req := range []string{"timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count"} { + if _, ok := m[req]; !ok { + t.Errorf("ultra-degraded tier: required field %q missing", req) + } + } + + // Optional fields must be absent + for _, opt := range []string{"targeting_key", "variant", "allocation", "targeting_rule", "error", "context", "runtime_default_used"} { + if _, ok := m[opt]; ok { + t.Errorf("ultra-degraded tier: optional field %q should be absent", opt) + } + } + }) + + t.Run("first_evaluation and last_evaluation meet minimum constraint", func(t *testing.T) { + // Schema minimum: 1759276800000 (2025-08-01 Unix ms) + // Using time.Now().UnixMilli() always satisfies this. + const schemaMin int64 = 1759276800000 + if nowMs < schemaMin { + t.Errorf("time.Now().UnixMilli() = %d is below schema minimum %d; use current timestamps only", nowMs, schemaMin) + } + }) + + t.Run("batch payload wraps events in flagEvaluations array", func(t *testing.T) { + payload := flagEvaluationPayload{ + Context: flagEvalDDContext{ + Service: "test-service", + Env: "test", + Version: "1.0.0", + }, + FlagEvaluations: []flagEvaluationEvent{ + { + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "test-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 1, + }, + }, + } + + b, err := json.Marshal(payload) + if err != nil { + t.Fatalf("failed to marshal batch payload: %v", err) + } + + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + if _, ok := m["context"]; !ok { + t.Error("batch payload: context missing") + } + if _, ok := m["flagEvaluations"]; !ok { + t.Error("batch payload: flagEvaluations array missing") + } + if arr, ok := m["flagEvaluations"].([]any); !ok || len(arr) != 1 { + t.Errorf("batch payload: expected 1 flagEvaluations entry, got %v", m["flagEvaluations"]) + } + }) +} + +// TestAggregatorCollisionSafety verifies that two distinct inputs that would collide +// under FNV-1a-only map keying land in SEPARATE buckets under the struct-keyed map +// (CONT-05 / source_comment_id 3395004724). +// +// It must fail RED: aggregator.add panics with "not implemented". +func TestAggregatorCollisionSafety(t *testing.T) { + agg := setupTestAggregator(t) + nowMs := time.Now().UnixMilli() + + // Two evaluations that differ only in allocationKey — they must be in separate full buckets. + // Under FNV-1a alone (on a concatenated string), carefully crafted keys can collide; + // under a struct-keyed map, these are structurally distinct and cannot collide. + d1 := evalDetails{ + flagKey: "my-flag", + variant: "on", + allocationKey: "alloc-a", + reason: "targeting_match", + } + d2 := evalDetails{ + flagKey: "my-flag", + variant: "on", + allocationKey: "alloc-b", + reason: "targeting_match", + } + + agg.add(d1, nil, nowMs) + agg.add(d2, nil, nowMs) + + agg.mu.Lock() + defer agg.mu.Unlock() + + if len(agg.full) != 2 { + t.Errorf("expected 2 separate full-tier buckets for distinct allocationKeys, got %d", len(agg.full)) + } + + // A second add with d1 must increment the existing bucket, not create a third + agg.mu.Unlock() + agg.add(d1, nil, nowMs) + agg.mu.Lock() + + if len(agg.full) != 2 { + t.Errorf("re-adding d1 must increment existing bucket, not create new one; got %d buckets", len(agg.full)) + } +} + +// TestAggregatorConcurrentMinMax verifies that 1000 goroutines recording the same key +// produce count==1000 and firstEvaluation<=lastEvaluation (CONT-06 / source_comment_id 3395176782). +// Must be run with -race to satisfy the race-free requirement. +func TestAggregatorConcurrentMinMax(t *testing.T) { + agg := &flagEvaluationAggregator{ + full: make(map[evaluationAggregationKey]*evaluationEntry), + degraded: make(map[evaluationDegradedKey]*evaluationEntry), + ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), + perFlagFull: make(map[string]int), + globalCap: 100_000, // large enough not to overflow during this test + perFlagCap: 100_000, + degradedCap: 100_000, + } + + d := evalDetails{ + flagKey: "concurrent-flag", + variant: "on", + reason: "targeting_match", + } + + const goroutines = 1000 + var wg sync.WaitGroup + wg.Add(goroutines) + + for range goroutines { + go func() { + defer wg.Done() + nowMs := time.Now().UnixMilli() + agg.add(d, nil, nowMs) + }() + } + wg.Wait() + + agg.mu.Lock() + defer agg.mu.Unlock() + + if len(agg.full) != 1 { + t.Fatalf("expected exactly 1 full-tier bucket, got %d", len(agg.full)) + } + + for _, entry := range agg.full { + if entry.count != goroutines { + t.Errorf("expected count=%d, got %d", goroutines, entry.count) + } + if entry.firstEvaluation > entry.lastEvaluation { + t.Errorf("firstEvaluation=%d > lastEvaluation=%d — min/max invariant violated", + entry.firstEvaluation, entry.lastEvaluation) + } + } +} + +// TestAggregatorCapOverflow verifies that: +// - Exceeding perFlagCap routes new entries to the degraded map (CONT-10). +// - Exceeding degradedCap routes new entries to the ultra-degraded map (CONT-10). +// - Global cap bounds total bucket growth (D-08 / source_comment_id 3385309427). +func TestAggregatorCapOverflow(t *testing.T) { + t.Run("perFlagCap overflow routes to degraded", func(t *testing.T) { + agg := setupTestAggregator(t) // perFlagCap=3 + nowMs := time.Now().UnixMilli() + + // Fill perFlagCap (3) full-tier buckets for "flag-a" + for i := range 3 { + d := evalDetails{ + flagKey: "flag-a", + variant: "on", + allocationKey: fmt.Sprintf("alloc-%d", i), + reason: "targeting_match", + targetingKey: fmt.Sprintf("user-%d", i), + } + agg.add(d, map[string]any{"key": fmt.Sprintf("v%d", i)}, nowMs) + } + + agg.mu.Lock() + if agg.perFlagFull["flag-a"] != 3 { + t.Errorf("expected perFlagFull[flag-a]=3, got %d", agg.perFlagFull["flag-a"]) + } + agg.mu.Unlock() + + // The 4th distinct entry for "flag-a" must overflow to degraded + d4 := evalDetails{ + flagKey: "flag-a", + variant: "on", + allocationKey: "alloc-overflow", + reason: "targeting_match", + targetingKey: "user-overflow", + } + agg.add(d4, map[string]any{"extra": "data"}, nowMs) + + agg.mu.Lock() + defer agg.mu.Unlock() + + if len(agg.degraded) == 0 { + t.Error("expected at least one degraded bucket after perFlagCap overflow") + } + }) + + t.Run("degradedCap overflow routes to ultra-degraded", func(t *testing.T) { + agg := setupTestAggregator(t) // degradedCap=3 + nowMs := time.Now().UnixMilli() + + // Pre-fill the degraded map to capacity by forcing overflow from full tier. + // Use different variants to get 3 distinct degraded buckets. + for i := range 4 { // 4 fills full to cap=3 then overflows once + for j := range 3 { // perFlagCap=3; 4 distinct allocs per flag => overflow on 4th + d := evalDetails{ + flagKey: fmt.Sprintf("flag-%d", i), + variant: fmt.Sprintf("v%d", j), + allocationKey: fmt.Sprintf("alloc-%d", j), + reason: "targeting_match", + targetingKey: fmt.Sprintf("user-%d", j), + } + agg.add(d, nil, nowMs) + } + } + + // Continue adding until degradedCap is also exhausted. + // At that point, ultra-degraded must receive new entries. + for i := range 10 { + d := evalDetails{ + flagKey: fmt.Sprintf("overflow-flag-%d", i), + variant: "on", + reason: "targeting_match", + targetingKey: fmt.Sprintf("user-%d", i), + } + // Force each into degraded by also filling its full tier + for j := range 4 { + d2 := evalDetails{ + flagKey: d.flagKey, + variant: d.variant, + allocationKey: fmt.Sprintf("a%d", j), + reason: d.reason, + } + agg.add(d2, nil, nowMs) + } + } + + agg.mu.Lock() + defer agg.mu.Unlock() + + if len(agg.ultraDeg) == 0 { + t.Error("expected ultra-degraded buckets after degradedCap overflow") + } + }) + + t.Run("globalCap bounds total bucket growth", func(t *testing.T) { + agg := setupTestAggregator(t) // globalCap=10 + nowMs := time.Now().UnixMilli() + + // Add many distinct evaluations; global count must not exceed globalCap + for i := range 50 { + d := evalDetails{ + flagKey: fmt.Sprintf("flag-%d", i), + variant: "on", + reason: "targeting_match", + } + agg.add(d, nil, nowMs) + } + + agg.mu.Lock() + defer agg.mu.Unlock() + + total := len(agg.full) + len(agg.degraded) + len(agg.ultraDeg) + if total > agg.globalCap { + t.Errorf("total buckets %d exceeds globalCap %d", total, agg.globalCap) + } + }) +} From 495f4c70105046b8d4f10b0d0fb9a0ff7eaa96f3 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 11 Jun 2026 22:02:10 -0400 Subject: [PATCH 02/37] feat(01-01): add BenchmarkFlagEvaluationNoop/OTelOnly/OTelPlusEVP benchmark scenarios - BenchmarkFlagEvaluationNoop: raw evaluator baseline (zero hooks) - BenchmarkFlagEvaluationOTelOnly: OTel hook only (existing Path A reference) - BenchmarkFlagEvaluationOTelPlusEVP: OTel + EVP hook scaffold (plan 02 wires the hook) - Each scenario has typical (100 flags, 50 users, 10-field context) and stress (10 flags, 1000 users, 250-field context) sub-benchmarks via b.Run - makeBenchmarkConfig() and makeBenchmarkContext() helpers added - All three compile and run with -benchtime=1x (CONT-08 scaffold) - EVP hook created as nil-writer stub in OTelPlusEVP; body not called until plan 02 wires it --- openfeature/provider_bench_test.go | 167 +++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 603d5e34ea1..4be390d1859 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -232,3 +232,170 @@ func BenchmarkConcurrentEvaluations(b *testing.B) { } }) } + +// makeBenchmarkConfig creates a test config with the specified number of flags. +// Extends createTestConfig() for benchmark load profiles. +func makeBenchmarkConfig(numFlags int) *universalFlagsConfiguration { + config := createTestConfig() + for i := len(config.Flags); i < numFlags; i++ { + flagKey := "bench-flag-" + string(rune('a'+i%26)) + string(rune('0'+i/26%10)) + config.Flags[flagKey] = &flag{ + Key: flagKey, + Enabled: true, + VariationType: valueTypeBoolean, + Variations: map[string]*variant{ + "on": {Key: "on", Value: true}, + }, + Allocations: []*allocation{ + { + Key: "alloc-bench", + Rules: []*rule{}, + Splits: []*split{ + {Shards: []*shard{}, VariationKey: "on"}, + }, + }, + }, + } + } + return config +} + +// makeBenchmarkContext creates a FlattenedContext with numFields attributes. +func makeBenchmarkContext(numFields int) openfeature.FlattenedContext { + ctx := openfeature.FlattenedContext{ + "targetingKey": "bench-user-001", + } + for i := 1; i < numFields; i++ { + ctx["field"+string(rune('a'+i%26))] = "value" + } + return ctx +} + +// BenchmarkFlagEvaluationNoop measures the raw evaluation cost with zero hooks — +// the pure evaluator baseline for the three-column overhead comparison (CONT-08 / D-11). +// +// Profile: typical (100 flags, 50-user simulation, 10-field context). +// Profile: stress (10 flags, 1000-user simulation, 250-field context — near degraded trigger). +// +// Run command: +// +// GOFLAGS=-mod=readonly go test ./openfeature -run='^$' -bench='^BenchmarkFlagEvaluation' \ +// -benchmem -count=3 -cpu=8 +func BenchmarkFlagEvaluationNoop(b *testing.B) { + profiles := []struct { + name string + numFlags int + numUsers int + numFields int + }{ + {"typical/100flags_50users_10fields", 100, 50, 10}, + {"stress/10flags_1000users_250fields", 10, 1000, 250}, + } + + for _, p := range profiles { + b.Run(p.name, func(b *testing.B) { + // Noop: provider with no hooks (nil out hooks after construction) + provider := newDatadogProvider(ProviderConfig{}) + provider.flagEvalHook = nil // no OTel hook + provider.exposureHook = nil // no exposure hook + config := makeBenchmarkConfig(p.numFlags) + provider.updateConfiguration(config) + + ctx := context.Background() + flatCtx := makeBenchmarkContext(p.numFields) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + // Rotate user targeting key to simulate p.numUsers distinct users + flatCtx["targetingKey"] = "bench-user-" + string(rune('0'+b.N%p.numUsers%10)) + _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) + } + }) + } +} + +// BenchmarkFlagEvaluationOTelOnly measures the evaluation cost with only the existing +// OTel feature_flag.evaluations hook (Path A — preserved baseline) (CONT-08 / D-11). +func BenchmarkFlagEvaluationOTelOnly(b *testing.B) { + profiles := []struct { + name string + numFlags int + numUsers int + numFields int + }{ + {"typical/100flags_50users_10fields", 100, 50, 10}, + {"stress/10flags_1000users_250fields", 10, 1000, 250}, + } + + for _, p := range profiles { + b.Run(p.name, func(b *testing.B) { + // OTel-only: provider with flagEvalHook (OTel) but no EVP hook + provider := newDatadogProvider(ProviderConfig{}) + provider.exposureHook = nil // no exposure hook — isolate OTel cost only + // provider.flagEvalHook is set by newDatadogProvider — keep it + config := makeBenchmarkConfig(p.numFlags) + provider.updateConfiguration(config) + + ctx := context.Background() + flatCtx := makeBenchmarkContext(p.numFields) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + flatCtx["targetingKey"] = "bench-user-" + string(rune('0'+b.N%p.numUsers%10)) + _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) + } + }) + } +} + +// BenchmarkFlagEvaluationOTelPlusEVP measures the marginal cost of adding the new EVP +// flagevaluation hook alongside the existing OTel hook (Path A + Path B) (CONT-08 / D-11). +// +// NOTE: The EVP hook is a signature-only stub in plan 01; the hook body panics with +// "not implemented". In this benchmark, the hook is constructed but NOT wired into +// provider.Hooks() — wiring is done in plan 02. This benchmark compiles and runs cleanly +// as a scaffold; plan 02 will wire the hook and the overhead numbers will be meaningful. +func BenchmarkFlagEvaluationOTelPlusEVP(b *testing.B) { + profiles := []struct { + name string + numFlags int + numUsers int + numFields int + }{ + {"typical/100flags_50users_10fields", 100, 50, 10}, + {"stress/10flags_1000users_250fields", 10, 1000, 250}, + } + + for _, p := range profiles { + b.Run(p.name, func(b *testing.B) { + // OTel+EVP: provider with OTel hook + EVP hook stub constructed. + // The EVP hook is created here to verify compilation; plan 02 wires it + // into provider.Hooks() and the aggregation buffer. + provider := newDatadogProvider(ProviderConfig{}) + provider.exposureHook = nil // isolate hook overhead + // provider.flagEvalHook is set by newDatadogProvider (OTel hook) + + // Construct the EVP hook stub — verifies the signature compiles. + // TODO(plan-02): wire evalHook into provider.flagEvalHook2 / Hooks(). + _ = newFlagEvaluationHook(nil) // nil writer — hook body panics; not called directly + + config := makeBenchmarkConfig(p.numFlags) + provider.updateConfiguration(config) + + ctx := context.Background() + flatCtx := makeBenchmarkContext(p.numFields) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + flatCtx["targetingKey"] = "bench-user-" + string(rune('0'+b.N%p.numUsers%10)) + _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) + } + }) + } +} From 5dac07f522df66c4926878db63ece3d52c597b46 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 11 Jun 2026 22:46:31 -0400 Subject: [PATCH 03/37] feat(01-02): implement flagEvaluationAggregator + writer + transport reuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - struct-keyed aggregation maps (collision-safe, CONT-05) - three-tier cascade: full → degraded → ultra-degraded (CONT-04/CONT-10) - first/last via min/max inside lock (CONT-06) - pruneContext 256-field/256-char limits before buffering (CONT-03) - phantom attempt counting: perFlagFull incremented even on globalCap drops so per-flag overflow path stays alive after full-tier cap is exhausted - reused exposure.go transport: UDS/HTTP branch, evpSubdomainHeader/Value (D-04) - dedicated 10s flush ticker, flush-on-shutdown, panic recovery (D-05) - FlagEvaluationFlushInterval added to ProviderConfig --- openfeature/flagevaluation.go | 466 ++++++++++++++++++++++++++++++++-- openfeature/provider.go | 4 + 2 files changed, 443 insertions(+), 27 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index b85465f981c..893a37c03a3 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -3,17 +3,31 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2025 Datadog, Inc. -// Package openfeature provides flag evaluation EVP emission. -// This file contains signature-only stubs; the real implementation is in plan 02. package openfeature import ( + "bytes" + "cmp" + "context" + "fmt" + "hash/fnv" + "io" + "log/slog" "net/http" "net/url" + "os" + "strings" "sync" "time" jsoniter "github.com/json-iterator/go" + + "github.com/DataDog/dd-trace-go/v2/internal" + "github.com/DataDog/dd-trace-go/v2/internal/env" + "github.com/DataDog/dd-trace-go/v2/internal/globalconfig" + "github.com/DataDog/dd-trace-go/v2/internal/log" + telemetrylog "github.com/DataDog/dd-trace-go/v2/internal/telemetry/log" + of "github.com/open-feature/go-sdk/openfeature" ) @@ -172,51 +186,449 @@ type evalDetails struct { } // newFlagEvaluationWriter creates a new flag evaluation writer. -// Plan 02 implements the body; this is a signature-only stub. -func newFlagEvaluationWriter(_ ProviderConfig) *flagEvaluationWriter { - panic("not implemented") +// The writer uses the same HTTP transport setup as exposure.go (D-04). +func newFlagEvaluationWriter(config ProviderConfig) *flagEvaluationWriter { + agentURL := internal.AgentURLFromEnv() + var httpClient *http.Client + if agentURL.Scheme == "unix" { + httpClient = internal.UDSClient(agentURL.Path, defaultHTTPTimeout) + agentURL = internal.UnixDataSocketURL(agentURL.Path) + } else { + httpClient = internal.DefaultHTTPClient(defaultHTTPTimeout, false) + } + + executable, _ := os.Executable() + + flushInterval := cmp.Or(config.FlagEvaluationFlushInterval, defaultFlagEvalFlushInterval) + + return &flagEvaluationWriter{ + flushInterval: flushInterval, + httpClient: httpClient, + agentURL: agentURL, + stopChan: make(chan struct{}), + jsonConfig: jsoniter.Config{}.Froze(), + ddContext: flagEvalDDContext{ + Service: cmp.Or(env.Get("DD_SERVICE"), globalconfig.ServiceName(), executable), + Version: env.Get("DD_VERSION"), + Env: env.Get("DD_ENV"), + }, + aggregator: flagEvaluationAggregator{ + full: make(map[evaluationAggregationKey]*evaluationEntry), + degraded: make(map[evaluationDegradedKey]*evaluationEntry), + ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), + perFlagFull: make(map[string]int), + globalCap: defaultEvalGlobalCap, + perFlagCap: defaultEvalPerFlagCap, + degradedCap: defaultEvalDegradedCap, + }, + } } // start begins the periodic flushing — called from InitWithContext(), NOT from the constructor. -// Plan 02 implements the body. +// Mirrors exposure.go's start() goroutine + panic recovery pattern. func (w *flagEvaluationWriter) start() { - panic("not implemented") + w.ticker = time.NewTicker(w.flushInterval) + go func() { + defer func() { + if r := recover(); r != nil { + log.Error("openfeature: flag evaluation writer recovered panic: %s", r) + var errAttr slog.Attr + if err, ok := r.(error); ok { + errAttr = slog.Any("panic", telemetrylog.NewSafeError(err)) + } else { + errAttr = slog.Any("panic", r) + } + telemetrylog.Error("openfeature: flag evaluation writer recovered panic", errAttr) + } + w.stop() + }() + + for { + select { + case <-w.ticker.C: + w.flush() + case <-w.stopChan: + return + } + } + }() } // stop stops the flush ticker and marks the writer as stopped. -// Plan 02 implements the body. func (w *flagEvaluationWriter) stop() { - panic("not implemented") + w.aggregator.mu.Lock() + if w.stopped { + w.aggregator.mu.Unlock() + return + } + w.stopped = true + w.aggregator.mu.Unlock() + + // Signal the goroutine to stop + close(w.stopChan) + + // Stop the ticker + if w.ticker != nil { + w.ticker.Stop() + } + + log.Debug("openfeature: flag evaluation writer stopped") } -// flush sends aggregated evaluation events to the Datadog Agent EVP proxy. -// Plan 02 implements the body. +// flush drains the aggregator, assembles per-tier events, and sends them to the agent. func (w *flagEvaluationWriter) flush() { - panic("not implemented") + w.aggregator.mu.Lock() + + // Under lock: drain all three maps. + full := w.aggregator.full + degraded := w.aggregator.degraded + ultraDeg := w.aggregator.ultraDeg + stopped := w.stopped + + if (len(full)+len(degraded)+len(ultraDeg)) == 0 || stopped { + w.aggregator.mu.Unlock() + return + } + + // Reset maps. + w.aggregator.full = make(map[evaluationAggregationKey]*evaluationEntry) + w.aggregator.degraded = make(map[evaluationDegradedKey]*evaluationEntry) + w.aggregator.ultraDeg = make(map[evaluationUltraDegradedKey]*evaluationEntry) + w.aggregator.perFlagFull = make(map[string]int) + w.aggregator.globalCount = 0 + + w.aggregator.mu.Unlock() + + nowMs := time.Now().UnixMilli() + var events []flagEvaluationEvent + + // Full tier events. + for key, e := range full { + ev := flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: key.flagKey}, + FirstEvaluation: e.firstEvaluation, + LastEvaluation: e.lastEvaluation, + EvaluationCount: e.count, + RuntimeDefault: e.runtimeDefault, + TargetingKey: e.targetingKey, + } + if key.variant != "" { + ev.Variant = &flagEvalVariant{Key: key.variant} + } + if key.allocationKey != "" { + ev.Allocation = &flagEvalAllocation{Key: key.allocationKey} + } + if key.targetingRuleKey != "" { + ev.TargetingRule = &flagEvalTargetingRule{Key: key.targetingRuleKey} + } + if e.errorMessage != "" { + ev.Error = &flagEvalError{Message: e.errorMessage} + } + if len(e.contextAttrs) > 0 { + ev.Context = &flagEvalEventContext{Evaluation: e.contextAttrs} + } + events = append(events, ev) + } + + // Degraded tier events (no targeting_key, no context.evaluation). + for key, e := range degraded { + ev := flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: key.flagKey}, + FirstEvaluation: e.firstEvaluation, + LastEvaluation: e.lastEvaluation, + EvaluationCount: e.count, + RuntimeDefault: e.runtimeDefault, + } + if key.variant != "" { + ev.Variant = &flagEvalVariant{Key: key.variant} + } + if key.allocationKey != "" { + ev.Allocation = &flagEvalAllocation{Key: key.allocationKey} + } + events = append(events, ev) + } + + // Ultra-degraded tier events (only required fields + flag + variant). + for key, e := range ultraDeg { + ev := flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: key.flagKey}, + FirstEvaluation: e.firstEvaluation, + LastEvaluation: e.lastEvaluation, + EvaluationCount: e.count, + } + if key.variant != "" { + ev.Variant = &flagEvalVariant{Key: key.variant} + } + events = append(events, ev) + } + + if len(events) == 0 { + return + } + + payload := flagEvaluationPayload{ + Context: w.ddContext, + FlagEvaluations: events, + } + + if err := w.sendToAgent(payload); err != nil { + log.Error("openfeature: failed to send flag evaluation events: %v", err.Error()) + } else { + log.Debug("openfeature: successfully sent %d flag evaluation events", len(events)) + } +} + +// record extracts evaluation details and adds them to the aggregation buffer. +// Called from the Finally hook after every evaluation. +func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.InterfaceEvaluationDetails) { + // Extract details. + d := extractEvalDetails(hookContext, details) + + // Flatten and prune the evaluation context. + attrs := hookContext.EvaluationContext().Attributes() + var contextAttrs map[string]any + if len(attrs) > 0 { + flattened := flattenContext(attrs) + contextAttrs = pruneContext(flattened) + } + + nowMs := time.Now().UnixMilli() + w.aggregator.add(d, contextAttrs, nowMs) +} + +// sendToAgent sends the flag evaluation payload to the Datadog Agent via EVP proxy. +// Reuses evpSubdomainHeader / evpSubdomainValue constants from exposure.go (D-04). +func (w *flagEvaluationWriter) sendToAgent(payload flagEvaluationPayload) error { + var bytesBuffer bytes.Buffer + encoder := w.jsonConfig.NewEncoder(&bytesBuffer) + if err := encoder.Encode(payload); err != nil { + return fmt.Errorf("failed to encode flag evaluation payload: %w", err) + } + + u := *w.agentURL + u.Path = flagEvaluationEndpoint + requestURL := u.String() + + req, err := http.NewRequestWithContext(context.Background(), "POST", requestURL, &bytesBuffer) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set(evpSubdomainHeader, evpSubdomainValue) + + log.Debug("openfeature: sending flag evaluation events to %s", requestURL) + + resp, err := w.httpClient.Do(req) + if err != nil { + return fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 256)) + return fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) + } + return nil +} + +// add records one evaluation observation into the appropriate aggregation tier. +// Must be called WITHOUT the aggregator lock held (it acquires the lock internally). +// Implements the three-tier cascade: full → degraded → ultra-degraded (CONT-04/CONT-10). +// +// Per-flag attempt counting: perFlagFull[flag] is incremented on every call for a flag +// (whether or not a full-tier bucket is actually created). This ensures that once +// globalCap is full, a flag that accumulates enough attempts (>= perFlagCap) still +// overflows to degraded — keeping the per-flag overflow path alive even after the +// global full-tier cap is exhausted. +func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]any, nowMs int64) { + a.mu.Lock() + defer a.mu.Unlock() + + // Build the full key using struct fields (collision-safe — CONT-05). + fullKey := evaluationAggregationKey{ + flagKey: d.flagKey, + variant: d.variant, + allocationKey: d.allocationKey, + targetingRuleKey: d.targetingRuleKey, + reason: d.reason, + targetingKey: d.targetingKey, + contextHash: hashContext(contextAttrs), + } + + // Check if this exact full-tier bucket already exists → fast-path increment. + if e, ok := a.full[fullKey]; ok { + e.count++ + if nowMs < e.firstEvaluation { + e.firstEvaluation = nowMs + } + if nowMs > e.lastEvaluation { + e.lastEvaluation = nowMs + } + return + } + + // Check per-flag cap. + if a.perFlagFull[d.flagKey] >= a.perFlagCap { + // perFlagCap exceeded — route to degraded tier. + a.addToDegraded(d, nowMs) + return + } + + // Per-flag cap not yet reached. Increment the attempt count for this flag + // regardless of whether we can actually create a full-tier bucket. This ensures + // the degraded overflow path activates correctly even when globalCap is full. + a.perFlagFull[d.flagKey]++ + + // Check globalCap before creating a new full-tier bucket. + if a.globalCount >= a.globalCap { + // Global cap full — attempt counted above; no new bucket created. + // The next attempt for this flag may eventually overflow perFlagCap → degraded. + return + } + + // New full-tier entry. + a.full[fullKey] = &evaluationEntry{ + count: 1, + firstEvaluation: nowMs, + lastEvaluation: nowMs, + runtimeDefault: d.runtimeDefault, + targetingKey: d.targetingKey, + contextAttrs: contextAttrs, + errorMessage: d.errorMessage, + } + a.globalCount++ +} + +// addToDegraded adds an entry to the degraded map (drops targeting_key + context). +// Called with the aggregator lock held. +// The degraded map is capped by degradedCap; overflow routes to ultra-degraded. +func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { + degKey := evaluationDegradedKey{ + flagKey: d.flagKey, + variant: d.variant, + allocationKey: d.allocationKey, + reason: d.reason, + } + + if e, ok := a.degraded[degKey]; ok { + e.count++ + if nowMs < e.firstEvaluation { + e.firstEvaluation = nowMs + } + if nowMs > e.lastEvaluation { + e.lastEvaluation = nowMs + } + return + } + + // New degraded bucket — check degradedCap. + if len(a.degraded) >= a.degradedCap { + // degradedCap exceeded — fall through to ultra-degraded. + a.addToUltraDegraded(d, nowMs) + return + } + + a.degraded[degKey] = &evaluationEntry{ + count: 1, + firstEvaluation: nowMs, + lastEvaluation: nowMs, + runtimeDefault: d.runtimeDefault, + } } -// record adds an evaluation to the aggregation buffer. -// Plan 02 implements the body. -func (w *flagEvaluationWriter) record(_ of.HookContext, _ of.InterfaceEvaluationDetails) { - panic("not implemented") +// addToUltraDegraded adds an entry to the ultra-degraded map (only flag key + variant). +// Called with the aggregator lock held. +// Ultra-degraded has no explicit cap — it is naturally bounded by the flag×variant +// enumeration (CONT-10). The globalCap enforcement applies only to the full tier. +func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64) { + ultraKey := evaluationUltraDegradedKey{ + flagKey: d.flagKey, + variant: d.variant, + } + + if e, ok := a.ultraDeg[ultraKey]; ok { + e.count++ + if nowMs < e.firstEvaluation { + e.firstEvaluation = nowMs + } + if nowMs > e.lastEvaluation { + e.lastEvaluation = nowMs + } + return + } + + // New ultra-degraded bucket — always created (naturally bounded by flag×variant). + a.ultraDeg[ultraKey] = &evaluationEntry{ + count: 1, + firstEvaluation: nowMs, + lastEvaluation: nowMs, + } } -// add adds one evaluation observation to the aggregator. -// Plan 02 implements the body. -func (a *flagEvaluationAggregator) add(_ evalDetails, _ map[string]any, _ int64) { - panic("not implemented") +// hashContext computes a FNV-1a hash of the pruned context map for use as a +// discriminator inside evaluationAggregationKey. The struct key itself is collision-safe +// across all enumerable dimensions; the hash supplements context identity only. +func hashContext(attrs map[string]any) uint64 { + if len(attrs) == 0 { + return 0 + } + h := fnv.New64a() + for k, v := range attrs { + _, _ = fmt.Fprintf(h, "%s=%v\n", k, v) + } + return h.Sum64() } // pruneContext applies 256-field / 256-char limits before buffering (D-07). // Mirrors worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH exactly. -// Plan 02 implements the body. -func pruneContext(_ map[string]any) map[string]any { - panic("not implemented") +// Must be called AFTER flattenContext() (from flatten.go) to expand nested objects first. +func pruneContext(raw map[string]any) map[string]any { + if len(raw) == 0 { + return nil + } + out := make(map[string]any, min(len(raw), maxContextFields)) + count := 0 + for k, v := range raw { + if count >= maxContextFields { + break + } + if s, ok := v.(string); ok && len(s) > maxFieldLength { + // Skip oversized string values (matches worker.ts pruneFields behavior). + continue + } + out[k] = v + count++ + } + if len(out) == 0 { + return nil + } + return out } // extractEvalDetails extracts EVP-relevant fields from hook context and evaluation details. -// Used only by flagEvaluationHook (flageval_metrics.go is NOT modified — D-06/PRES-01). -// Plan 02 implements the body. -func extractEvalDetails(_ of.HookContext, _ of.InterfaceEvaluationDetails) evalDetails { - panic("not implemented") +// This helper is used only by flagEvaluationHook — it does NOT replace the extraction in +// flageval_metrics.go (that file is untouched per D-06/PRES-01). +func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluationDetails) evalDetails { + allocationKey, _ := details.FlagMetadata[metadataAllocationKey].(string) + reason := strings.ToLower(string(details.Reason)) + if reason == "" { + reason = "unknown" + } + var errMsg string + if details.ErrorCode != "" { + errMsg = string(details.ErrorCode) + } + return evalDetails{ + flagKey: hookContext.FlagKey(), + variant: details.Variant, + reason: reason, + allocationKey: allocationKey, + targetingKey: hookContext.EvaluationContext().TargetingKey(), + errorMessage: errMsg, + runtimeDefault: isRuntimeDefault(details), + } } diff --git a/openfeature/provider.go b/openfeature/provider.go index 78bda66b7bc..b561e2c9f1f 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -45,6 +45,10 @@ type ProviderConfig struct { // ExposureFlushInterval is the interval at which exposure events are flushed to the agent // Default: 1 second ExposureFlushInterval time.Duration + + // FlagEvaluationFlushInterval is the interval for flushing EVP flag evaluation events. + // Default: 10 seconds (D-05 contract value). Leave zero to use the default. + FlagEvaluationFlushInterval time.Duration } // DatadogProvider is an OpenFeature provider that evaluates feature flags From 35d5d9eb68451dc2d1cfe896c50a62d14aa802aa Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 11 Jun 2026 22:46:45 -0400 Subject: [PATCH 04/37] feat(01-02): implement flagEvaluationHook Finally stage + extraction helpers - flagEvaluationHook.Finally: ctx.Done check, nil-guard, calls writer.record - isRuntimeDefault: variant=="" primary signal; DEFAULT/DISABLED reason secondary (CONT-07) - extractEvalDetails: shared helper for EVP hook only; flageval_metrics.go untouched (D-06/PRES-01) --- openfeature/flagevaluation_hook.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/openfeature/flagevaluation_hook.go b/openfeature/flagevaluation_hook.go index 87695e7fbad..fb8390c7da2 100644 --- a/openfeature/flagevaluation_hook.go +++ b/openfeature/flagevaluation_hook.go @@ -3,8 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2025 Datadog, Inc. -// Package openfeature provides the EVP flag evaluation OpenFeature hook. -// This file contains signature-only stubs; the real implementation is in plan 02. package openfeature import ( @@ -28,21 +26,31 @@ func newFlagEvaluationHook(w *flagEvaluationWriter) *flagEvaluationHook { // Finally is called after every flag evaluation (success or error). // Using Finally (not After) ensures error-path and provider-not-ready evaluations are counted (CONT-09). -// Plan 02 implements the body; this is a signature-only stub. +// Mirrors flageval_metrics.go's Finally stage; the EVP hook is a separate registered hook (D-06). func (h *flagEvaluationHook) Finally( ctx context.Context, hookContext of.HookContext, details of.InterfaceEvaluationDetails, _ of.HookHints, ) { - panic("not implemented") + select { + case <-ctx.Done(): + return + default: + } + if h.writer == nil { + return + } + h.writer.record(hookContext, details) } // isRuntimeDefault returns true when the caller's supplied default value was returned. // Primary signal: absent variant key (flag-not-found, provider-not-ready, type-mismatch, no allocation). // Secondary: explicit DEFAULT or DISABLED reason (belt-and-suspenders). // Satisfies CONT-07 (source_comment_id 3395344504). -// Plan 02 implements the body; this is a signature-only stub. -func isRuntimeDefault(_ of.InterfaceEvaluationDetails) bool { - panic("not implemented") +func isRuntimeDefault(details of.InterfaceEvaluationDetails) bool { + if details.Variant == "" { + return true + } + return details.Reason == of.DefaultReason || details.Reason == of.DisabledReason } From 4b44f575617e7858caf0b43350d142c931b6434a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 11 Jun 2026 22:54:18 -0400 Subject: [PATCH 05/37] feat(01-02): wire EVP flagevaluation hook + writer into provider (Task 3) - Add DD_FLAGGING_EVALUATION_COUNTS_ENABLED killswitch to provider (default true) - Register flagEvaluationWriter + flagEvaluationHook in newDatadogProvider when enabled - Start/flush/stop writer in InitWithContext / ShutdownWithContext - Append EVP hook in Hooks() alongside existing OTel flagEvalHook (PRES-01 preserved) - Add FlagEvaluationFlushInterval to ProviderConfig - Register DD_FLAGGING_EVALUATION_COUNTS_ENABLED in supported_configurations.gen.go - Add TestFlagEvaluationKillswitch: 3 subtests verifying killswitch behaviour - Update TestNewDatadogProvider to expect 3 hooks (exposure + OTel + EVP) --- internal/env/supported_configurations.gen.go | 1 + internal/env/supported_configurations.json | 7 ++ openfeature/flagevaluation_provider_test.go | 110 +++++++++++++++++++ openfeature/provider.go | 41 ++++++- openfeature/provider_test.go | 6 +- 5 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 openfeature/flagevaluation_provider_test.go diff --git a/internal/env/supported_configurations.gen.go b/internal/env/supported_configurations.gen.go index ff9fb07a233..248f66be7a9 100644 --- a/internal/env/supported_configurations.gen.go +++ b/internal/env/supported_configurations.gen.go @@ -67,6 +67,7 @@ var SupportedConfigurations = map[string]struct{}{ "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": {}, "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED": {}, "DD_EXTERNAL_ENV": {}, + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": {}, "DD_GIT_BRANCH": {}, "DD_GIT_COMMIT_AUTHOR_DATE": {}, "DD_GIT_COMMIT_AUTHOR_EMAIL": {}, diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index 71c2c744aaa..ac8dfa51539 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -413,6 +413,13 @@ "default": null } ], + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": [ + { + "implementation": "A", + "type": "FIX_ME", + "default": "FIX_ME" + } + ], "DD_GIT_BRANCH": [ { "implementation": "A", diff --git a/openfeature/flagevaluation_provider_test.go b/openfeature/flagevaluation_provider_test.go new file mode 100644 index 00000000000..c37e065d137 --- /dev/null +++ b/openfeature/flagevaluation_provider_test.go @@ -0,0 +1,110 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +package openfeature + +import ( + "reflect" + "testing" + + of "github.com/open-feature/go-sdk/openfeature" +) + +// TestFlagEvaluationKillswitch verifies that DD_FLAGGING_EVALUATION_COUNTS_ENABLED (default true) +// controls ONLY the EVP flagevaluation hook/writer, leaving the OTel flagEvalHook unaffected (PRES-01). +// +// When the killswitch is "false": the EVP hook (flagEvalEVPHook) is NOT registered in Hooks() +// and flagEvalWriter is nil. +// When the killswitch is unset or "true": the EVP hook IS registered. +// The OTel flagEvalHook is present in Hooks() in BOTH cases. +func TestFlagEvaluationKillswitch(t *testing.T) { + t.Run("killswitch disabled: EVP hook absent from Hooks(), OTel hook present", func(t *testing.T) { + t.Setenv(flagEvalCountsEnabledEnvVar, "false") + + p := newDatadogProvider(ProviderConfig{}) + + if p.flagEvalWriter != nil { + t.Error("expected flagEvalWriter to be nil when killswitch is disabled") + } + if p.flagEvalEVPHook != nil { + t.Error("expected flagEvalEVPHook to be nil when killswitch is disabled") + } + + hooks := p.Hooks() + + // OTel hook must still be present (PRES-01). + otelPresent := false + for _, h := range hooks { + if _, ok := h.(*flagEvalHook); ok { + otelPresent = true + } + } + if !otelPresent { + t.Error("expected OTel flagEvalHook to be present in Hooks() even when killswitch is disabled") + } + + // EVP hook must be absent. + for _, h := range hooks { + if _, ok := h.(*flagEvaluationHook); ok { + t.Errorf("expected EVP flagEvaluationHook to be absent from Hooks() when killswitch is disabled, but found one: %v", reflect.TypeOf(h)) + } + } + }) + + t.Run("killswitch enabled (unset = default true): EVP hook present in Hooks()", func(t *testing.T) { + // Ensure the env var is unset to test the default-true behavior. + t.Setenv(flagEvalCountsEnabledEnvVar, "1") + + p := newDatadogProvider(ProviderConfig{}) + + if p.flagEvalWriter == nil { + t.Error("expected flagEvalWriter to be non-nil when killswitch is enabled (default)") + } + if p.flagEvalEVPHook == nil { + t.Error("expected flagEvalEVPHook to be non-nil when killswitch is enabled (default)") + } + + hooks := p.Hooks() + + // Both OTel and EVP hooks must be present. + otelPresent := false + evpPresent := false + for _, h := range hooks { + switch h.(type) { + case *flagEvalHook: + otelPresent = true + case *flagEvaluationHook: + evpPresent = true + } + } + if !otelPresent { + t.Error("expected OTel flagEvalHook to be present in Hooks() when killswitch is enabled") + } + if !evpPresent { + t.Error("expected EVP flagEvaluationHook to be present in Hooks() when killswitch is enabled") + } + }) + + t.Run("killswitch explicitly true: EVP hook present in Hooks()", func(t *testing.T) { + t.Setenv(flagEvalCountsEnabledEnvVar, "true") + + p := newDatadogProvider(ProviderConfig{}) + + hooks := p.Hooks() + + evpPresent := false + for _, h := range hooks { + if _, ok := h.(*flagEvaluationHook); ok { + evpPresent = true + } + } + if !evpPresent { + t.Error("expected EVP flagEvaluationHook to be present in Hooks() when killswitch is explicitly 'true'") + } + }) +} + +// Compile-time assertion: flagEvaluationHook implements the OpenFeature Hook interface. +var _ of.Hook = (*flagEvaluationHook)(nil) diff --git a/openfeature/provider.go b/openfeature/provider.go index b561e2c9f1f..9c4d3dcd252 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -34,6 +34,11 @@ var ( const ( // ffeProductEnvVar is the environment variable to enable the experimental flagging provider ffeProductEnvVar = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED" + // flagEvalCountsEnabledEnvVar is the operator killswitch for the EVP flagevaluation emission path. + // Default: true (EVP path is ON by default). Set to "false" to disable only the EVP path + // while leaving the OTel feature_flag.evaluations path unaffected (PRES-01). + // Mirrors the internal.BoolEnv convention used by ffeProductEnvVar. + flagEvalCountsEnabledEnvVar = "DD_FLAGGING_EVALUATION_COUNTS_ENABLED" // Default timeout for provider initialization defaultInitTimeout = 30 * time.Second // Default timeout for provider shutdown @@ -66,6 +71,12 @@ type DatadogProvider struct { // Flag evaluation metrics hook (OTel counter via Finally hook) flagEvalHook *flagEvalHook + + // Flag evaluation EVP writer + hook (new Path B — EVP flagevaluation track). + // Both fields are nil when DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false (killswitch). + // Named distinctly from flagEvalHook (OTel) to avoid collisions. + flagEvalWriter *flagEvaluationWriter + flagEvalEVPHook *flagEvaluationHook } // NewDatadogProvider creates a new Datadog OpenFeature provider with default configuration. @@ -107,6 +118,17 @@ func newDatadogProvider(config ProviderConfig) *DatadogProvider { exposureHook: hook, flagEvalHook: newFlagEvalHook(metrics), } + + // Conditionally construct the EVP flagevaluation writer + hook. + // Gated by DD_FLAGGING_EVALUATION_COUNTS_ENABLED (default true). + // When false, both fields are left nil and the EVP path is disabled. + // The OTel hook (flagEvalHook above) is registered unconditionally (PRES-01). + if internal.BoolEnv(flagEvalCountsEnabledEnvVar, true) { + evalWriter := newFlagEvaluationWriter(config) + p.flagEvalWriter = evalWriter + p.flagEvalEVPHook = newFlagEvaluationHook(evalWriter) + } + p.configChange.L = &p.mu return p } @@ -187,8 +209,12 @@ func (p *DatadogProvider) InitWithContext(ctx context.Context, _ openfeature.Eva } } - // Start periodic flushing + // Start periodic flushing for exposure writer. p.exposureWriter.start() + // Start periodic flushing for EVP flag evaluation writer (nil when killswitch disabled). + if p.flagEvalWriter != nil { + p.flagEvalWriter.start() + } return nil } @@ -219,6 +245,11 @@ func (p *DatadogProvider) ShutdownWithContext(ctx context.Context) error { p.exposureWriter.flush() p.exposureWriter.stop() } + // Stop the EVP flag evaluation writer (nil when killswitch disabled). + if p.flagEvalWriter != nil { + p.flagEvalWriter.flush() + p.flagEvalWriter.stop() + } // Shut down flag evaluation metrics if p.flagEvalHook != nil && p.flagEvalHook.metrics != nil { _ = p.flagEvalHook.metrics.shutdown(ctx) @@ -413,15 +444,21 @@ func (p *DatadogProvider) ObjectEvaluation( } // Hooks returns the hooks for this provider. -// This includes the exposure tracking hook and the flag evaluation metrics hook. +// This includes the exposure tracking hook, the OTel flag evaluation metrics hook (always), +// and the EVP flagevaluation hook (only when DD_FLAGGING_EVALUATION_COUNTS_ENABLED is true). func (p *DatadogProvider) Hooks() []openfeature.Hook { var hooks []openfeature.Hook if p.exposureHook != nil { hooks = append(hooks, p.exposureHook) } + // OTel hook is always registered — untouched by the EVP killswitch (PRES-01). if p.flagEvalHook != nil { hooks = append(hooks, p.flagEvalHook) } + // EVP hook is nil when the killswitch disabled it. + if p.flagEvalEVPHook != nil { + hooks = append(hooks, p.flagEvalEVPHook) + } return hooks } diff --git a/openfeature/provider_test.go b/openfeature/provider_test.go index 76ee7e217c6..1e5f088ce1d 100644 --- a/openfeature/provider_test.go +++ b/openfeature/provider_test.go @@ -255,8 +255,10 @@ func TestNewDatadogProvider(t *testing.T) { } hooks := provider.Hooks() - if len(hooks) != 2 { - t.Errorf("expected 2 hooks (exposure + flag eval metrics), got %d", len(hooks)) + // 3 hooks: exposure hook + OTel flag eval metrics hook + EVP flagevaluation hook. + // The EVP hook is enabled by default (DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true). + if len(hooks) != 3 { + t.Errorf("expected 3 hooks (exposure + flag eval metrics + EVP flagevaluation), got %d", len(hooks)) } } From 4b75012243b7f39db49a15063d0bb5ed656f88d5 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 11 Jun 2026 23:06:52 -0400 Subject: [PATCH 06/37] feat(01-03): wire real EVP hook in OTelPlusEVP benchmark (no-flush writer) - BenchmarkFlagEvaluationOTelPlusEVP now uses ProviderConfig.FlagEvaluationFlushInterval=24h to construct a real EVP writer+hook without HTTP round-trips in the hot path (Pitfall 4) - Removed the plan-01 stub comment and nil-writer construction - Added time import for 24h duration constant --- openfeature/provider_bench_test.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 4be390d1859..0789174659b 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -8,6 +8,7 @@ package openfeature import ( "context" "testing" + "time" "github.com/open-feature/go-sdk/openfeature" ) @@ -355,10 +356,8 @@ func BenchmarkFlagEvaluationOTelOnly(b *testing.B) { // BenchmarkFlagEvaluationOTelPlusEVP measures the marginal cost of adding the new EVP // flagevaluation hook alongside the existing OTel hook (Path A + Path B) (CONT-08 / D-11). // -// NOTE: The EVP hook is a signature-only stub in plan 01; the hook body panics with -// "not implemented". In this benchmark, the hook is constructed but NOT wired into -// provider.Hooks() — wiring is done in plan 02. This benchmark compiles and runs cleanly -// as a scaffold; plan 02 will wire the hook and the overhead numbers will be meaningful. +// The EVP writer uses a 24-hour flush interval (effectively infinite) so no HTTP round-trip +// is in the hot path — this benchmarks only the hook + aggregator cost (RESEARCH Pitfall 4). func BenchmarkFlagEvaluationOTelPlusEVP(b *testing.B) { profiles := []struct { name string @@ -372,16 +371,13 @@ func BenchmarkFlagEvaluationOTelPlusEVP(b *testing.B) { for _, p := range profiles { b.Run(p.name, func(b *testing.B) { - // OTel+EVP: provider with OTel hook + EVP hook stub constructed. - // The EVP hook is created here to verify compilation; plan 02 wires it - // into provider.Hooks() and the aggregation buffer. - provider := newDatadogProvider(ProviderConfig{}) - provider.exposureHook = nil // isolate hook overhead - // provider.flagEvalHook is set by newDatadogProvider (OTel hook) - - // Construct the EVP hook stub — verifies the signature compiles. - // TODO(plan-02): wire evalHook into provider.flagEvalHook2 / Hooks(). - _ = newFlagEvaluationHook(nil) // nil writer — hook body panics; not called directly + // OTel+EVP: provider with both OTel hook and EVP hook wired. + // The EVP writer uses a 24-hour flush interval so no HTTP round-trip + // is in the hot path — benchmarks hook + aggregator cost only. + provider := newDatadogProvider(ProviderConfig{ + FlagEvaluationFlushInterval: 24 * time.Hour, // never flush during bench + }) + provider.exposureHook = nil // isolate hook overhead; no exposure cost config := makeBenchmarkConfig(p.numFlags) provider.updateConfiguration(config) From 83848c3105d6a85cd62ec83b6f7bf533a3e0721b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 00:03:26 -0400 Subject: [PATCH 07/37] fix(openfeature): deterministic context hash + drop dead targeting_rule field (CR-01, WR-01, WR-02) - CR-01: hashContext now hashes over sorted keys so identical contexts produce a stable FNV-1a hash; Go map iteration is randomized and was fragmenting buckets and corrupting evaluation_count. - WR-01: remove the never-populated targetingRuleKey field, its aggregation-key member, the flagEvalTargetingRule type, and the dead targeting_rule emission branch (no metadata source exists yet). - WR-02: correct the globalCap comment to state it bounds the full tier only (degraded/ultra-degraded are bounded separately). --- openfeature/flagevaluation.go | 103 +++++++++++++++-------------- openfeature/flagevaluation_test.go | 13 ++-- 2 files changed, 61 insertions(+), 55 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 893a37c03a3..a567781fba2 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -16,6 +16,7 @@ import ( "net/http" "net/url" "os" + "sort" "strings" "sync" "time" @@ -44,7 +45,7 @@ const ( maxFieldLength = 256 // Aggregation caps (D-08/D-09/CONT-10). - defaultEvalGlobalCap = 65_536 // bounds total distinct buckets across all tiers + defaultEvalGlobalCap = 65_536 // bounds full-tier buckets only; degraded/ultra are bounded separately (WR-02) defaultEvalPerFlagCap = 10_000 // bounds full-fidelity buckets per flag defaultEvalDegradedCap = 10_000 // bounds degraded map (absent from POC — Gate 8 fix) ) @@ -53,13 +54,12 @@ const ( // contextHash is a uint64 FNV-1a hash of the pruned context — NOT the sole map key. // Replaces the FNV-1a-keyed map from PR #4874 POC (CONT-05 / source_comment_id 3395004724). type evaluationAggregationKey struct { - flagKey string - variant string - allocationKey string - targetingRuleKey string - reason string - targetingKey string - contextHash uint64 // context is map[string]any (not comparable); hash is a discriminator only + flagKey string + variant string + allocationKey string + reason string + targetingKey string + contextHash uint64 // context is map[string]any (not comparable); hash is a discriminator only } // evaluationDegradedKey is the key for the degraded aggregation map. @@ -106,38 +106,42 @@ type flagEvaluationAggregator struct { // flagEvaluationEvent matches flagevaluation.json — required fields always present; // optional fields use omitempty (absent = schema-valid for degraded/ultra-degraded tiers). type flagEvaluationEvent struct { - Timestamp int64 `json:"timestamp"` - Flag flagEvalFlag `json:"flag"` - FirstEvaluation int64 `json:"first_evaluation"` - LastEvaluation int64 `json:"last_evaluation"` - EvaluationCount int64 `json:"evaluation_count"` - RuntimeDefault bool `json:"runtime_default_used,omitempty"` - TargetingKey string `json:"targeting_key,omitempty"` - Variant *flagEvalVariant `json:"variant,omitempty"` - Allocation *flagEvalAllocation `json:"allocation,omitempty"` - TargetingRule *flagEvalTargetingRule `json:"targeting_rule,omitempty"` - Error *flagEvalError `json:"error,omitempty"` - Context *flagEvalEventContext `json:"context,omitempty"` + Timestamp int64 `json:"timestamp"` + Flag flagEvalFlag `json:"flag"` + FirstEvaluation int64 `json:"first_evaluation"` + LastEvaluation int64 `json:"last_evaluation"` + EvaluationCount int64 `json:"evaluation_count"` + RuntimeDefault bool `json:"runtime_default_used,omitempty"` + TargetingKey string `json:"targeting_key,omitempty"` + Variant *flagEvalVariant `json:"variant,omitempty"` + Allocation *flagEvalAllocation `json:"allocation,omitempty"` + Error *flagEvalError `json:"error,omitempty"` + Context *flagEvalEventContext `json:"context,omitempty"` } // flagEvalFlag holds the flag key. -type flagEvalFlag struct{ Key string `json:"key"` } +type flagEvalFlag struct { + Key string `json:"key"` +} // flagEvalVariant holds the variant key. -type flagEvalVariant struct{ Key string `json:"key"` } +type flagEvalVariant struct { + Key string `json:"key"` +} // flagEvalAllocation holds the allocation key. -type flagEvalAllocation struct{ Key string `json:"key"` } - -// flagEvalTargetingRule holds the targeting rule key. -type flagEvalTargetingRule struct{ Key string `json:"key"` } +type flagEvalAllocation struct { + Key string `json:"key"` +} // flagEvalError holds the error message. -type flagEvalError struct{ Message string `json:"message"` } +type flagEvalError struct { + Message string `json:"message"` +} // flagEvalEventContext holds the per-event context object. type flagEvalEventContext struct { - Evaluation map[string]any `json:"evaluation,omitempty"` + Evaluation map[string]any `json:"evaluation,omitempty"` DD *flagEvalContextDD `json:"dd,omitempty"` } @@ -175,14 +179,13 @@ type flagEvaluationWriter struct { // evalDetails holds extracted flag evaluation fields for EVP aggregation. // Used only by flagEvaluationHook; does NOT replace extraction in flageval_metrics.go. type evalDetails struct { - flagKey string - variant string - reason string - allocationKey string - targetingRuleKey string - targetingKey string - errorMessage string - runtimeDefault bool + flagKey string + variant string + reason string + allocationKey string + targetingKey string + errorMessage string + runtimeDefault bool } // newFlagEvaluationWriter creates a new flag evaluation writer. @@ -319,9 +322,6 @@ func (w *flagEvaluationWriter) flush() { if key.allocationKey != "" { ev.Allocation = &flagEvalAllocation{Key: key.allocationKey} } - if key.targetingRuleKey != "" { - ev.TargetingRule = &flagEvalTargetingRule{Key: key.targetingRuleKey} - } if e.errorMessage != "" { ev.Error = &flagEvalError{Message: e.errorMessage} } @@ -450,13 +450,12 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an // Build the full key using struct fields (collision-safe — CONT-05). fullKey := evaluationAggregationKey{ - flagKey: d.flagKey, - variant: d.variant, - allocationKey: d.allocationKey, - targetingRuleKey: d.targetingRuleKey, - reason: d.reason, - targetingKey: d.targetingKey, - contextHash: hashContext(contextAttrs), + flagKey: d.flagKey, + variant: d.variant, + allocationKey: d.allocationKey, + reason: d.reason, + targetingKey: d.targetingKey, + contextHash: hashContext(contextAttrs), } // Check if this exact full-tier bucket already exists → fast-path increment. @@ -576,9 +575,17 @@ func hashContext(attrs map[string]any) uint64 { if len(attrs) == 0 { return 0 } + // Hash over a deterministic key ordering. Go map iteration is randomized, + // and FNV-1a is order-sensitive, so ranging the map directly would produce a + // different hash for identical contexts and fragment aggregation buckets (CR-01). + keys := make([]string, 0, len(attrs)) + for k := range attrs { + keys = append(keys, k) + } + sort.Strings(keys) h := fnv.New64a() - for k, v := range attrs { - _, _ = fmt.Fprintf(h, "%s=%v\n", k, v) + for _, k := range keys { + _, _ = fmt.Fprintf(h, "%s=%v\n", k, attrs[k]) } return h.Sum64() } diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 70b8eccb65b..a9944ac67a2 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -431,10 +431,9 @@ func TestAggregatorCapOverflow(t *testing.T) { // At that point, ultra-degraded must receive new entries. for i := range 10 { d := evalDetails{ - flagKey: fmt.Sprintf("overflow-flag-%d", i), - variant: "on", - reason: "targeting_match", - targetingKey: fmt.Sprintf("user-%d", i), + flagKey: fmt.Sprintf("overflow-flag-%d", i), + variant: "on", + reason: "targeting_match", } // Force each into degraded by also filling its full tier for j := range 4 { @@ -463,9 +462,9 @@ func TestAggregatorCapOverflow(t *testing.T) { // Add many distinct evaluations; global count must not exceed globalCap for i := range 50 { d := evalDetails{ - flagKey: fmt.Sprintf("flag-%d", i), - variant: "on", - reason: "targeting_match", + flagKey: fmt.Sprintf("flag-%d", i), + variant: "on", + reason: "targeting_match", } agg.add(d, nil, nowMs) } From 4dd9408dcd6507011f69e5b417bb218b74a183fc Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 00:03:33 -0400 Subject: [PATCH 08/37] fix(openfeature): count cancelled-context evaluations in Finally hook (WR-03) record() is a non-blocking in-memory add with no network call, so gating it on ctx.Done() silently dropped legitimate evaluation counts when the request context was already cancelled (undercut Gate-7). Remove the ctx.Done() guard and update TestFlagEvaluationHookContextCancelled to assert the evaluation IS still counted. --- openfeature/flagevaluation_hook.go | 12 ++++++------ openfeature/flagevaluation_hook_test.go | 11 +++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/openfeature/flagevaluation_hook.go b/openfeature/flagevaluation_hook.go index fb8390c7da2..50019cf2d64 100644 --- a/openfeature/flagevaluation_hook.go +++ b/openfeature/flagevaluation_hook.go @@ -28,16 +28,16 @@ func newFlagEvaluationHook(w *flagEvaluationWriter) *flagEvaluationHook { // Using Finally (not After) ensures error-path and provider-not-ready evaluations are counted (CONT-09). // Mirrors flageval_metrics.go's Finally stage; the EVP hook is a separate registered hook (D-06). func (h *flagEvaluationHook) Finally( - ctx context.Context, + _ context.Context, hookContext of.HookContext, details of.InterfaceEvaluationDetails, _ of.HookHints, ) { - select { - case <-ctx.Done(): - return - default: - } + // Do NOT gate buffering on the evaluation context. In real servers ctx is + // frequently the request context, which may already be cancelled by the time + // Finally runs — and record() is a non-blocking in-memory add with no network + // call. Gating on ctx.Done() would silently drop legitimate evaluation counts + // (undercuts Gate-7 for cancelled-request evals) (WR-03). if h.writer == nil { return } diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index ca7244abe46..7131d04b03f 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -214,10 +214,9 @@ func TestFlagEvaluationHookFinally(t *testing.T) { }) } -// TestFlagEvaluationHookContextCancelled verifies that a cancelled context causes -// Finally to return without recording (context safety / no zombie writes). -// -// It must fail RED: the hook's Finally method panics with "not implemented". +// TestFlagEvaluationHookContextCancelled verifies that a cancelled context does NOT +// drop the evaluation: record() is a non-blocking in-memory add, so a cancelled +// request context must still be counted (Gate-7 / WR-03). func TestFlagEvaluationHookContextCancelled(t *testing.T) { w := setupTestWriter(t) hook := newFlagEvaluationHook(w) @@ -234,7 +233,7 @@ func TestFlagEvaluationHookContextCancelled(t *testing.T) { defer w.aggregator.mu.Unlock() total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) - if total != 0 { - t.Errorf("expected no entries when context is cancelled, got %d", total) + if total != 1 { + t.Errorf("expected the cancelled-context evaluation to still be counted, got %d entries", total) } } From 58a96c3027113e641ed2c9ed88b7aa93475e1b0d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 00:03:39 -0400 Subject: [PATCH 09/37] fix(openfeature): rotate benchmark targeting key per iteration (WR-04) The benchmark used b.N inside a b.Loop() body, which is a single fixed total (not a per-iteration index), so every evaluation used an identical targeting key and the 50/1000-user cardinality profiles collapsed to one bucket. Use a per-iteration counter with strconv.Itoa(i%numUsers) so the Gate-6 aggregation cost is exercised. --- openfeature/provider_bench_test.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 0789174659b..dd5bfd6518b 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -7,6 +7,7 @@ package openfeature import ( "context" + "strconv" "testing" "time" @@ -297,8 +298,8 @@ func BenchmarkFlagEvaluationNoop(b *testing.B) { b.Run(p.name, func(b *testing.B) { // Noop: provider with no hooks (nil out hooks after construction) provider := newDatadogProvider(ProviderConfig{}) - provider.flagEvalHook = nil // no OTel hook - provider.exposureHook = nil // no exposure hook + provider.flagEvalHook = nil // no OTel hook + provider.exposureHook = nil // no exposure hook config := makeBenchmarkConfig(p.numFlags) provider.updateConfiguration(config) @@ -308,9 +309,13 @@ func BenchmarkFlagEvaluationNoop(b *testing.B) { b.ReportAllocs() b.ResetTimer() + i := 0 for b.Loop() { - // Rotate user targeting key to simulate p.numUsers distinct users - flatCtx["targetingKey"] = "bench-user-" + string(rune('0'+b.N%p.numUsers%10)) + // Rotate user targeting key to simulate p.numUsers distinct users. + // Use a per-iteration counter (not b.N, which is a single fixed total + // under b.Loop()) so the cardinality profile is actually exercised (WR-04). + flatCtx["targetingKey"] = "bench-user-" + strconv.Itoa(i%p.numUsers) + i++ _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) } }) @@ -345,8 +350,10 @@ func BenchmarkFlagEvaluationOTelOnly(b *testing.B) { b.ReportAllocs() b.ResetTimer() + i := 0 for b.Loop() { - flatCtx["targetingKey"] = "bench-user-" + string(rune('0'+b.N%p.numUsers%10)) + flatCtx["targetingKey"] = "bench-user-" + strconv.Itoa(i%p.numUsers) + i++ _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) } }) @@ -388,8 +395,10 @@ func BenchmarkFlagEvaluationOTelPlusEVP(b *testing.B) { b.ReportAllocs() b.ResetTimer() + i := 0 for b.Loop() { - flatCtx["targetingKey"] = "bench-user-" + string(rune('0'+b.N%p.numUsers%10)) + flatCtx["targetingKey"] = "bench-user-" + strconv.Itoa(i%p.numUsers) + i++ _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) } }) From ff23b7ca40edb5886b37e6948ecbca049138b957 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 00:03:44 -0400 Subject: [PATCH 10/37] chore(env): set DD_FLAGGING_EVALUATION_COUNTS_ENABLED type/default and regenerate The supported_configurations.json entry had placeholder FIX_ME values for type and default, failing validate_supported_configurations_v2_local_file. Set type=boolean, default=true (killswitch defaults on) and regenerate supported_configurations.gen.go via scripts/configinverter so the JSON and generated map are canonical and in sync. --- internal/env/supported_configurations.gen.go | 2 +- internal/env/supported_configurations.json | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/env/supported_configurations.gen.go b/internal/env/supported_configurations.gen.go index 248f66be7a9..231669bda93 100644 --- a/internal/env/supported_configurations.gen.go +++ b/internal/env/supported_configurations.gen.go @@ -149,6 +149,7 @@ var SupportedConfigurations = map[string]struct{}{ "DD_TEST_OPTIMIZATION_MANIFEST_FILE": {}, "DD_TEST_OPTIMIZATION_PAYLOADS_IN_FILES": {}, "DD_TEST_SESSION_NAME": {}, + "DD_TRACER_EXPERIMENTAL_SPAN_POOL_ENABLED": {}, "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED": {}, "DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED": {}, "DD_TRACE_ABANDONED_SPAN_TIMEOUT": {}, @@ -238,7 +239,6 @@ var SupportedConfigurations = map[string]struct{}{ "DD_TRACE_SEND_RETRIES": {}, "DD_TRACE_SOURCE_HOSTNAME": {}, "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA": {}, - "DD_TRACER_EXPERIMENTAL_SPAN_POOL_ENABLED": {}, "DD_TRACE_SQL_ANALYTICS_ENABLED": {}, "DD_TRACE_SQL_COMMENT_INJECTION_MODE": {}, "DD_TRACE_STARTUP_LOGS": {}, diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index ac8dfa51539..afc27fe8da2 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -416,8 +416,8 @@ "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": [ { "implementation": "A", - "type": "FIX_ME", - "default": "FIX_ME" + "type": "boolean", + "default": "true" } ], "DD_GIT_BRANCH": [ @@ -987,6 +987,13 @@ "default": null } ], + "DD_TRACER_EXPERIMENTAL_SPAN_POOL_ENABLED": [ + { + "implementation": "A", + "type": "boolean", + "default": "false" + } + ], "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED": [ { "implementation": "A", @@ -1610,13 +1617,6 @@ "default": "v0" } ], - "DD_TRACER_EXPERIMENTAL_SPAN_POOL_ENABLED": [ - { - "implementation": "A", - "type": "boolean", - "default": "false" - } - ], "DD_TRACE_SQL_ANALYTICS_ENABLED": [ { "implementation": "A", From 81da9e19456c0ed941b1fc9f785168d165241d2d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 01:13:55 -0400 Subject: [PATCH 11/37] fix(openfeature): preserve count signal on global-cap overflow; align EVP flagevaluation contract - #1 (blocker): route globalCap overflow to ultra-degraded instead of dropping; count total preserved at every tier - #3: add TestSaturationCountPreservation (total counts == add() calls past both globalCap and perFlagCap) - #2: correct collision-safe wording to low-collision/not collision-proof; document count-preserving collision behavior - #4: rotate flag keys + distinct context field names in benchmark so real cardinality is exercised --- openfeature/flagevaluation.go | 71 ++++++++++++++++--- openfeature/flagevaluation_test.go | 110 +++++++++++++++++++++++++++-- openfeature/provider_bench_test.go | 16 +++-- 3 files changed, 174 insertions(+), 23 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index a567781fba2..15f1617434e 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -50,9 +50,44 @@ const ( defaultEvalDegradedCap = 10_000 // bounds degraded map (absent from POC — Gate 8 fix) ) -// evaluationAggregationKey is a struct-keyed map key (collision-free for enumerable dims). -// contextHash is a uint64 FNV-1a hash of the pruned context — NOT the sole map key. -// Replaces the FNV-1a-keyed map from PR #4874 POC (CONT-05 / source_comment_id 3395004724). +// evaluationAggregationKey identifies one full-tier aggregation bucket. Its identity is +// split into two parts with very different collision properties: +// +// 1. Enumerable dimensions — flagKey, variant, allocationKey, reason, targetingKey — are +// stored as EXACT string fields and compared byte-for-byte by Go map equality. Two +// structurally distinct values can NEVER alias on these dimensions. In particular, a +// count can never be attributed to the wrong flag/variant/reason/allocation. +// +// 2. contextHash is a 64-bit FNV-1a digest of the pruned evaluation context (see +// hashContext, hashed over sorted keys for determinism). The context is map[string]any +// and therefore not comparable, so it cannot be a struct field — the digest stands in +// for it. The digest is deterministic and low-collision but NOT collision-proof. +// +// What a contextHash collision actually does — and why it is acceptable (D-13 / CONT-05): +// +// A collision requires two evaluations that are identical on ALL FIVE exact dimensions +// AND whose *different* contexts happen to hash to the same uint64. When that occurs, the +// second evaluation matches the existing bucket on add()'s fast path and increments its +// count (the `if e, ok := a.full[fullKey]; ok` branch below). Consequences: +// +// - COUNT-PRESERVING: the evaluation count is merged into the existing bucket, never +// dropped. The invariant Σ(counts across all tiers) == number of add() calls still +// holds. This is exactly what TestSaturationCountPreservation guards. +// - NO MISATTRIBUTION: the count still belongs to the correct flag/variant/reason/ +// allocation — those dimensions are exact, not hashed. +// - SOLE CASUALTY is context-attribute fidelity: the bucket retains the FIRST context's +// attrs, so the colliding evaluation's distinct attrs are not separately reported. +// Context is a best-effort dimension that the degraded/ultra tiers drop entirely by +// design, so a collision is strictly less lossy than ordinary degradation. +// +// Probability is bounded by the cap: the full tier holds at most globalCap (65536) +// buckets, so there are <= ~65k distinct digests per flush window. Birthday bound +// ~= 65536^2 / 2^65 ~= 1.2e-10 per window. This is internal telemetry over the customer's +// own context — not a trust boundary — so a keyed/crypto hash (e.g. SipHash) is +// unwarranted, and a wider (128-bit) digest would buy only context-label fidelity at that +// 1e-10 tail. Deliberately not done. +// +// (CONT-05 / source_comment_id 3395004724; resolution recorded as decision D-13.) type evaluationAggregationKey struct { flagKey string variant string @@ -448,7 +483,7 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an a.mu.Lock() defer a.mu.Unlock() - // Build the full key using struct fields (collision-safe — CONT-05). + // Build the full key: exact struct fields for enumerable dims + 64-bit FNV context digest (CONT-05). fullKey := evaluationAggregationKey{ flagKey: d.flagKey, variant: d.variant, @@ -458,7 +493,14 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an contextHash: hashContext(contextAttrs), } - // Check if this exact full-tier bucket already exists → fast-path increment. + // Fast path: this exact full-tier bucket already exists → increment its count. + // + // This branch is ALSO where a contextHash collision lands (see the + // evaluationAggregationKey doc): if two evaluations share all five exact dimensions and + // their differing contexts hash equal, the second matches here and merges into the + // first's bucket. The count is preserved — never dropped, never misattributed; only the + // second context's distinct attrs are not separately recorded. This is the + // count-preserving guarantee TestSaturationCountPreservation asserts. if e, ok := a.full[fullKey]; ok { e.count++ if nowMs < e.firstEvaluation { @@ -484,8 +526,11 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an // Check globalCap before creating a new full-tier bucket. if a.globalCount >= a.globalCap { - // Global cap full — attempt counted above; no new bucket created. - // The next attempt for this flag may eventually overflow perFlagCap → degraded. + // Global cap full — count must not be lost (CONT-10/D-08/D-09). + // Route into ultra-degraded (flag key + counts only) so the count signal is preserved. + // The per-flag attempt counter was already incremented above; once it reaches + // perFlagCap this flag will route through addToDegraded instead. + a.addToUltraDegraded(d, nowMs) return } @@ -568,9 +613,15 @@ func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64 } } -// hashContext computes a FNV-1a hash of the pruned context map for use as a -// discriminator inside evaluationAggregationKey. The struct key itself is collision-safe -// across all enumerable dimensions; the hash supplements context identity only. +// hashContext computes a deterministic 64-bit FNV-1a hash of the pruned context map for +// use as a discriminator inside evaluationAggregationKey. The enumerable struct fields +// (flagKey, variant, allocationKey, reason, targetingKey) are exact string comparisons and +// cannot collide; contextHash is a probabilistic supplement for context identity only — +// it is low-collision but NOT collision-proof (CONT-05). +// +// A digest collision is count-preserving (it merges into an existing bucket via add()'s +// fast path, costing only context-attribute fidelity, never a count) — see the +// evaluationAggregationKey doc and decision D-13 for the full collision analysis. func hashContext(attrs map[string]any) uint64 { if len(attrs) == 0 { return 0 diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index a9944ac67a2..b2bc6e4b876 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -363,6 +363,79 @@ func TestAggregatorConcurrentMinMax(t *testing.T) { } } +// TestSaturationCountPreservation is the regression guard for BLOCKER #1 (silent drop at +// globalCap overflow). It asserts that the sum of all evaluation counts across ALL tiers +// (full + degraded + ultra-degraded) equals the total number of add() calls, even after +// BOTH globalCap AND perFlagCap have been exhausted. +// +// This test MUST FAIL on the pre-fix code (negative control proving the silent drop), and +// MUST PASS after rerouting the globalCap-overflow return into the ultra-degraded tier +// (CONT-10 / D-08 / D-09 / source_comment_id 3385309423). +func TestSaturationCountPreservation(t *testing.T) { + // Use small caps so we can saturate them quickly. + // globalCap=5 means only 5 full-tier buckets ever created. + // perFlagCap=2 means after 2 distinct full-tier buckets per flag, it overflows to degraded. + // degradedCap=3 means only 3 degraded buckets; further overflow goes to ultra-degraded. + agg := &flagEvaluationAggregator{ + full: make(map[evaluationAggregationKey]*evaluationEntry), + degraded: make(map[evaluationDegradedKey]*evaluationEntry), + ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), + perFlagFull: make(map[string]int), + globalCap: 5, + perFlagCap: 2, + degradedCap: 3, + } + nowMs := time.Now().UnixMilli() + + // We drive 100 distinct evaluations. Each call to add() must contribute exactly 1 + // count unit to one of the three tiers. After all calls, the Σ must equal 100. + // + // Strategy: use 20 different flag keys × 5 distinct allocationKey combos so that: + // - The first 2 per flag go into the full tier (perFlagCap=2). + // - The next ones overflow to degraded (bounded by degradedCap=3). + // - Once degraded is also full, overflow to ultra-degraded. + // - Once globalCap(5) is hit, any flag's attempt-count not yet at perFlagCap routes + // through the globalCap branch — the BLOCKER is that this branch returns silently + // instead of routing to ultra-degraded. + const totalCalls = 100 + for i := range totalCalls { + flagIdx := i % 20 + allocIdx := i % 5 + d := evalDetails{ + flagKey: fmt.Sprintf("sat-flag-%d", flagIdx), + variant: "on", + allocationKey: fmt.Sprintf("alloc-%d", allocIdx), + reason: "targeting_match", + targetingKey: fmt.Sprintf("user-%d", i%10), + } + agg.add(d, nil, nowMs) + } + + // Sum counts across all three tiers. + agg.mu.Lock() + defer agg.mu.Unlock() + + var totalCounted int64 + for _, e := range agg.full { + totalCounted += e.count + } + for _, e := range agg.degraded { + totalCounted += e.count + } + for _, e := range agg.ultraDeg { + totalCounted += e.count + } + + if totalCounted != totalCalls { + t.Errorf( + "count preservation violated: Σ(full+degraded+ultraDeg)=%d, expected=%d (add() calls); "+ + "silent drops detected (full buckets=%d, degraded buckets=%d, ultraDeg buckets=%d)", + totalCounted, totalCalls, + len(agg.full), len(agg.degraded), len(agg.ultraDeg), + ) + } +} + // TestAggregatorCapOverflow verifies that: // - Exceeding perFlagCap routes new entries to the degraded map (CONT-10). // - Exceeding degradedCap routes new entries to the ultra-degraded map (CONT-10). @@ -455,12 +528,17 @@ func TestAggregatorCapOverflow(t *testing.T) { } }) - t.Run("globalCap bounds total bucket growth", func(t *testing.T) { - agg := setupTestAggregator(t) // globalCap=10 + t.Run("globalCap bounds full-tier bucket growth only", func(t *testing.T) { + agg := setupTestAggregator(t) // globalCap=10, perFlagCap=3, degradedCap=3 nowMs := time.Now().UnixMilli() - // Add many distinct evaluations; global count must not exceed globalCap - for i := range 50 { + // Add 50 distinct evaluations (each a unique flag key). + // globalCap=10 caps the full tier; overflow goes to ultra-degraded. + // The full tier must stay at or below globalCap. + // The total count across all tiers must equal the number of add() calls + // (no silent drops — CONT-10/D-08/D-09). + const calls = 50 + for i := range calls { d := evalDetails{ flagKey: fmt.Sprintf("flag-%d", i), variant: "on", @@ -472,9 +550,27 @@ func TestAggregatorCapOverflow(t *testing.T) { agg.mu.Lock() defer agg.mu.Unlock() - total := len(agg.full) + len(agg.degraded) + len(agg.ultraDeg) - if total > agg.globalCap { - t.Errorf("total buckets %d exceeds globalCap %d", total, agg.globalCap) + // Full tier must be bounded by globalCap. + if agg.globalCount > agg.globalCap { + t.Errorf("full-tier globalCount %d exceeds globalCap %d", agg.globalCount, agg.globalCap) + } + if len(agg.full) > agg.globalCap { + t.Errorf("full-tier buckets %d exceeds globalCap %d", len(agg.full), agg.globalCap) + } + + // Every add() call must have produced a count unit in some tier (no silent drops). + var totalCounted int64 + for _, e := range agg.full { + totalCounted += e.count + } + for _, e := range agg.degraded { + totalCounted += e.count + } + for _, e := range agg.ultraDeg { + totalCounted += e.count + } + if totalCounted != calls { + t.Errorf("count preservation violated: Σ(all tiers)=%d, expected=%d", totalCounted, calls) } }) } diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index dd5bfd6518b..231427bb875 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -138,9 +138,9 @@ func BenchmarkEvaluationWithVaryingContextSize(b *testing.B) { "country": "US", } - // Add additional fields to the context + // Add additional fields with unique names so all numFields are distinct. for i := 1; i < size.numFields; i++ { - flatCtx[string(rune('a'+i))] = i + flatCtx["field"+strconv.Itoa(i)] = i } b.ReportAllocs() @@ -170,9 +170,9 @@ func BenchmarkEvaluationWithVaryingFlagCounts(b *testing.B) { provider := newDatadogProvider(ProviderConfig{}) config := createTestConfig() - // Add additional flags + // Add additional flags with unique monotonic keys so cardinality is exercised. for i := len(config.Flags); i < count.numFlags; i++ { - flagKey := string(rune('a' + i)) + flagKey := "flag-" + strconv.Itoa(i) config.Flags[flagKey] = &flag{ Key: flagKey, Enabled: true, @@ -237,10 +237,12 @@ func BenchmarkConcurrentEvaluations(b *testing.B) { // makeBenchmarkConfig creates a test config with the specified number of flags. // Extends createTestConfig() for benchmark load profiles. +// Flag keys are unique monotonic integers ("bench-flag-N") so the claimed cardinality +// is fully exercised without wrapping (WR-04). func makeBenchmarkConfig(numFlags int) *universalFlagsConfiguration { config := createTestConfig() for i := len(config.Flags); i < numFlags; i++ { - flagKey := "bench-flag-" + string(rune('a'+i%26)) + string(rune('0'+i/26%10)) + flagKey := "bench-flag-" + strconv.Itoa(i) config.Flags[flagKey] = &flag{ Key: flagKey, Enabled: true, @@ -263,12 +265,14 @@ func makeBenchmarkConfig(numFlags int) *universalFlagsConfiguration { } // makeBenchmarkContext creates a FlattenedContext with numFields attributes. +// Field names are "fieldN" with a monotonic index so all numFields are distinct and +// the claimed context cardinality is fully exercised without wrapping (WR-04). func makeBenchmarkContext(numFields int) openfeature.FlattenedContext { ctx := openfeature.FlattenedContext{ "targetingKey": "bench-user-001", } for i := 1; i < numFields; i++ { - ctx["field"+string(rune('a'+i%26))] = "value" + ctx["field"+strconv.Itoa(i)] = "value" } return ctx } From cf520aa8aad3db486682a0b41c9cf44208884735 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 07:51:41 -0400 Subject: [PATCH 12/37] refactor(openfeature): async best-effort EVP flagevaluation recording; benchmark + comment fixes The OpenFeature Go SDK runs hooks synchronously on the caller's eval goroutine (client.go invokes finallyHooks via defer, no goroutine), so aggregating inside Finally added +112%/+180% latency over OTel-only. Move the work off the hot path. - async recording: Finally now does only cheap scalar extraction + the SDK's shallow Attributes() copy + a non-blocking bounded enqueue; a single background worker owns flatten/prune/hash/aggregate/flush. Queue-full drops with an observable counter (best-effort); stop() drains + final-flushes. Hot-path cost is a flat 5 allocs (~0.65us small / ~6us large context). - hashContext: replace per-field fmt.Fprintf with direct byte writes (250-field hash drops from ~250 allocs to ~1). - benchmark: drive evaluations through openfeature.Client so the hooks actually run (direct provider calls bypassed Hooks(), so prior overhead numbers measured nothing); isolate OTel-only vs OTel+EVP; rotate flag + user; add a hot-path-only micro-bench. - tests: drain the async queue in the hook tests; add TestFlagEvaluationBackpressureDrops. - comments: remove internal planning identifiers (this reference SDK is human-facing). --- openfeature/flagevaluation.go | 168 +++++++++++--- openfeature/flagevaluation_hook.go | 9 +- openfeature/flagevaluation_hook_test.go | 47 +++- openfeature/flagevaluation_provider_test.go | 4 +- openfeature/flagevaluation_test.go | 28 ++- openfeature/provider.go | 8 +- openfeature/provider_bench_test.go | 238 +++++++++++--------- 7 files changed, 328 insertions(+), 174 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 15f1617434e..9a0aed05bb9 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -17,8 +17,10 @@ import ( "net/url" "os" "sort" + "strconv" "strings" "sync" + "sync/atomic" "time" jsoniter "github.com/json-iterator/go" @@ -34,20 +36,25 @@ import ( const ( // defaultFlagEvalFlushInterval is the flush interval for EVP flag evaluation events. - // D-05: dedicated 10 s timer (contract value); separate from exposureWriter's 1 s interval. + // Dedicated 10 s timer; separate from exposureWriter's 1 s interval. defaultFlagEvalFlushInterval = 10 * time.Second // flagEvaluationEndpoint is the EVP proxy endpoint for flag evaluation events. flagEvaluationEndpoint = "/evp_proxy/v2/api/v2/flagevaluations" - // Context pruning limits (D-07) — mirror worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH. + // Context pruning limits — mirror worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH. maxContextFields = 256 maxFieldLength = 256 - // Aggregation caps (D-08/D-09/CONT-10). - defaultEvalGlobalCap = 65_536 // bounds full-tier buckets only; degraded/ultra are bounded separately (WR-02) + // Aggregation caps. + defaultEvalGlobalCap = 65_536 // bounds full-tier buckets only; degraded/ultra are bounded separately defaultEvalPerFlagCap = 10_000 // bounds full-fidelity buckets per flag - defaultEvalDegradedCap = 10_000 // bounds degraded map (absent from POC — Gate 8 fix) + defaultEvalDegradedCap = 10_000 // bounds degraded map + + // defaultEvalEventBufferSize bounds the async hand-off queue between the (hot-path) + // Finally hook and the background aggregation worker. On overflow the hook drops the + // event and increments a counter rather than blocking the evaluation. + defaultEvalEventBufferSize = 4096 ) // evaluationAggregationKey identifies one full-tier aggregation bucket. Its identity is @@ -63,7 +70,7 @@ const ( // and therefore not comparable, so it cannot be a struct field — the digest stands in // for it. The digest is deterministic and low-collision but NOT collision-proof. // -// What a contextHash collision actually does — and why it is acceptable (D-13 / CONT-05): +// What a contextHash collision actually does — and why it is acceptable: // // A collision requires two evaluations that are identical on ALL FIVE exact dimensions // AND whose *different* contexts happen to hash to the same uint64. When that occurs, the @@ -86,8 +93,6 @@ const ( // own context — not a trust boundary — so a keyed/crypto hash (e.g. SipHash) is // unwarranted, and a wider (128-bit) digest would buy only context-label fidelity at that // 1e-10 tail. Deliberately not done. -// -// (CONT-05 / source_comment_id 3395004724; resolution recorded as decision D-13.) type evaluationAggregationKey struct { flagKey string variant string @@ -209,6 +214,24 @@ type flagEvaluationWriter struct { stopChan chan struct{} stopped bool jsonConfig jsoniter.API + + // Asynchronous hand-off: the Finally hook enqueues a cheap snapshot here; a single + // background worker (started in start()) drains it and performs flatten/prune/hash/ + // aggregate off the evaluation hot path. events is bounded; on overflow the hook drops + // the event and bumps dropped — best-effort telemetry that never blocks the request. + events chan evalEvent + dropped atomic.Int64 + workerDone chan struct{} +} + +// evalEvent is the minimal snapshot the Finally hook hands to the worker. attrs is the +// owned copy returned by EvaluationContext().Attributes(), so it is safe to read off the +// hot path for scalar values (a nested mutable attribute the caller mutates after the call +// returns is a documented best-effort edge). +type evalEvent struct { + d evalDetails + attrs map[string]any + nowMs int64 } // evalDetails holds extracted flag evaluation fields for EVP aggregation. @@ -224,7 +247,7 @@ type evalDetails struct { } // newFlagEvaluationWriter creates a new flag evaluation writer. -// The writer uses the same HTTP transport setup as exposure.go (D-04). +// The writer uses the same HTTP transport setup as exposure.go. func newFlagEvaluationWriter(config ProviderConfig) *flagEvaluationWriter { agentURL := internal.AgentURLFromEnv() var httpClient *http.Client @@ -244,6 +267,8 @@ func newFlagEvaluationWriter(config ProviderConfig) *flagEvaluationWriter { httpClient: httpClient, agentURL: agentURL, stopChan: make(chan struct{}), + workerDone: make(chan struct{}), + events: make(chan evalEvent, defaultEvalEventBufferSize), jsonConfig: jsoniter.Config{}.Froze(), ddContext: flagEvalDDContext{ Service: cmp.Or(env.Get("DD_SERVICE"), globalconfig.ServiceName(), executable), @@ -278,14 +303,20 @@ func (w *flagEvaluationWriter) start() { } telemetrylog.Error("openfeature: flag evaluation writer recovered panic", errAttr) } - w.stop() + // Always signal completion so stop() unblocks, even on panic. + close(w.workerDone) }() + // Single owner of flatten/prune/hash/aggregate/flush. The hot path only enqueues; + // all that cost lives here, off the evaluation path. for { select { + case ev := <-w.events: + w.aggregate(ev) case <-w.ticker.C: w.flush() case <-w.stopChan: + w.drainAndFlush() return } } @@ -302,11 +333,12 @@ func (w *flagEvaluationWriter) stop() { w.stopped = true w.aggregator.mu.Unlock() - // Signal the goroutine to stop + // Signal the worker to drain the queue and do a final flush. close(w.stopChan) - - // Stop the ticker if w.ticker != nil { + // Worker was started: wait for its final flush before returning, then stop the + // ticker. (ticker is set only in start(), so it gates "was the worker started".) + <-w.workerDone w.ticker.Stop() } @@ -315,15 +347,19 @@ func (w *flagEvaluationWriter) stop() { // flush drains the aggregator, assembles per-tier events, and sends them to the agent. func (w *flagEvaluationWriter) flush() { + // Surface best-effort backpressure drops (queue full) as an observable signal. + if d := w.dropped.Swap(0); d > 0 { + log.Warn("openfeature: flag evaluation queue full — dropped %d evaluation(s) under backpressure (best-effort telemetry)", d) + } + w.aggregator.mu.Lock() // Under lock: drain all three maps. full := w.aggregator.full degraded := w.aggregator.degraded ultraDeg := w.aggregator.ultraDeg - stopped := w.stopped - if (len(full)+len(degraded)+len(ultraDeg)) == 0 || stopped { + if (len(full) + len(degraded) + len(ultraDeg)) == 0 { w.aggregator.mu.Unlock() return } @@ -416,26 +452,51 @@ func (w *flagEvaluationWriter) flush() { } } -// record extracts evaluation details and adds them to the aggregation buffer. -// Called from the Finally hook after every evaluation. +// record runs on the evaluation hot path (the Finally hook). It does only cheap scalar +// extraction plus the SDK's shallow context copy, then a non-blocking enqueue — no +// flatten/prune/hash/aggregation happens here; the background worker does that. If the +// queue is full the event is dropped and counted (best-effort), never blocking the +// evaluation. Called from the Finally hook after every evaluation. func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.InterfaceEvaluationDetails) { - // Extract details. - d := extractEvalDetails(hookContext, details) + ev := evalEvent{ + d: extractEvalDetails(hookContext, details), + attrs: hookContext.EvaluationContext().Attributes(), // SDK returns an owned copy + nowMs: time.Now().UnixMilli(), + } + select { + case w.events <- ev: + default: + w.dropped.Add(1) + } +} - // Flatten and prune the evaluation context. - attrs := hookContext.EvaluationContext().Attributes() +// aggregate performs the deferred flatten/prune/hash and updates the aggregator. It runs +// only on the writer's single worker goroutine. +func (w *flagEvaluationWriter) aggregate(ev evalEvent) { var contextAttrs map[string]any - if len(attrs) > 0 { - flattened := flattenContext(attrs) + if len(ev.attrs) > 0 { + flattened := flattenContext(ev.attrs) contextAttrs = pruneContext(flattened) } + w.aggregator.add(ev.d, contextAttrs, ev.nowMs) +} - nowMs := time.Now().UnixMilli() - w.aggregator.add(d, contextAttrs, nowMs) +// drainAndFlush processes any buffered events and performs a final flush. Called by the +// worker when stopping so a final batch is not lost on shutdown. +func (w *flagEvaluationWriter) drainAndFlush() { + for { + select { + case ev := <-w.events: + w.aggregate(ev) + default: + w.flush() + return + } + } } // sendToAgent sends the flag evaluation payload to the Datadog Agent via EVP proxy. -// Reuses evpSubdomainHeader / evpSubdomainValue constants from exposure.go (D-04). +// Reuses evpSubdomainHeader / evpSubdomainValue constants from exposure.go. func (w *flagEvaluationWriter) sendToAgent(payload flagEvaluationPayload) error { var bytesBuffer bytes.Buffer encoder := w.jsonConfig.NewEncoder(&bytesBuffer) @@ -472,7 +533,7 @@ func (w *flagEvaluationWriter) sendToAgent(payload flagEvaluationPayload) error // add records one evaluation observation into the appropriate aggregation tier. // Must be called WITHOUT the aggregator lock held (it acquires the lock internally). -// Implements the three-tier cascade: full → degraded → ultra-degraded (CONT-04/CONT-10). +// Implements the three-tier cascade: full → degraded → ultra-degraded. // // Per-flag attempt counting: perFlagFull[flag] is incremented on every call for a flag // (whether or not a full-tier bucket is actually created). This ensures that once @@ -483,7 +544,7 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an a.mu.Lock() defer a.mu.Unlock() - // Build the full key: exact struct fields for enumerable dims + 64-bit FNV context digest (CONT-05). + // Build the full key: exact struct fields for enumerable dims + 64-bit FNV context digest. fullKey := evaluationAggregationKey{ flagKey: d.flagKey, variant: d.variant, @@ -526,7 +587,7 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an // Check globalCap before creating a new full-tier bucket. if a.globalCount >= a.globalCap { - // Global cap full — count must not be lost (CONT-10/D-08/D-09). + // Global cap full — count must not be lost. // Route into ultra-degraded (flag key + counts only) so the count signal is preserved. // The per-flag attempt counter was already incremented above; once it reaches // perFlagCap this flag will route through addToDegraded instead. @@ -587,7 +648,7 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { // addToUltraDegraded adds an entry to the ultra-degraded map (only flag key + variant). // Called with the aggregator lock held. // Ultra-degraded has no explicit cap — it is naturally bounded by the flag×variant -// enumeration (CONT-10). The globalCap enforcement applies only to the full tier. +// enumeration. The globalCap enforcement applies only to the full tier. func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64) { ultraKey := evaluationUltraDegradedKey{ flagKey: d.flagKey, @@ -617,31 +678,66 @@ func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64 // use as a discriminator inside evaluationAggregationKey. The enumerable struct fields // (flagKey, variant, allocationKey, reason, targetingKey) are exact string comparisons and // cannot collide; contextHash is a probabilistic supplement for context identity only — -// it is low-collision but NOT collision-proof (CONT-05). +// it is low-collision but NOT collision-proof. // // A digest collision is count-preserving (it merges into an existing bucket via add()'s // fast path, costing only context-attribute fidelity, never a count) — see the -// evaluationAggregationKey doc and decision D-13 for the full collision analysis. +// evaluationAggregationKey doc for the full collision analysis. func hashContext(attrs map[string]any) uint64 { if len(attrs) == 0 { return 0 } // Hash over a deterministic key ordering. Go map iteration is randomized, // and FNV-1a is order-sensitive, so ranging the map directly would produce a - // different hash for identical contexts and fragment aggregation buckets (CR-01). + // different hash for identical contexts and fragment aggregation buckets. keys := make([]string, 0, len(attrs)) for k := range attrs { keys = append(keys, k) } sort.Strings(keys) h := fnv.New64a() + // Reuse one scratch buffer across fields and write the bytes directly. FNV's Write + // neither retains the slice nor allocates, so hashing an N-field context costs ~1 + // allocation instead of the per-field reflection allocation of fmt.Fprintf — the + // dominant per-evaluation cost of this hook for large contexts. + var buf []byte for _, k := range keys { - _, _ = fmt.Fprintf(h, "%s=%v\n", k, attrs[k]) + buf = buf[:0] + buf = append(buf, k...) + buf = append(buf, '=') + buf = appendContextValue(buf, attrs[k]) + buf = append(buf, '\n') + _, _ = h.Write(buf) } return h.Sum64() } -// pruneContext applies 256-field / 256-char limits before buffering (D-07). +// appendContextValue appends a deterministic string form of v to buf, avoiding allocation +// for the common scalar types; rare/complex types fall back to fmt. The hash is only an +// in-memory, per-flush-window discriminator, so the exact formatting is irrelevant as long +// as it is deterministic within a run. +func appendContextValue(buf []byte, v any) []byte { + switch x := v.(type) { + case string: + return append(buf, x...) + case bool: + return strconv.AppendBool(buf, x) + case int: + return strconv.AppendInt(buf, int64(x), 10) + case int64: + return strconv.AppendInt(buf, x, 10) + case int32: + return strconv.AppendInt(buf, int64(x), 10) + case float64: + return strconv.AppendFloat(buf, x, 'g', -1, 64) + case float32: + return strconv.AppendFloat(buf, float64(x), 'g', -1, 32) + default: + return append(buf, fmt.Sprintf("%v", x)...) + } +} + +// pruneContext applies 256-field / 256-char limits before buffering. // Mirrors worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH exactly. // Must be called AFTER flattenContext() (from flatten.go) to expand nested objects first. func pruneContext(raw map[string]any) map[string]any { @@ -669,7 +765,7 @@ func pruneContext(raw map[string]any) map[string]any { // extractEvalDetails extracts EVP-relevant fields from hook context and evaluation details. // This helper is used only by flagEvaluationHook — it does NOT replace the extraction in -// flageval_metrics.go (that file is untouched per D-06/PRES-01). +// flageval_metrics.go (that file is left untouched to preserve the OTel path). func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluationDetails) evalDetails { allocationKey, _ := details.FlagMetadata[metadataAllocationKey].(string) reason := strings.ToLower(string(details.Reason)) diff --git a/openfeature/flagevaluation_hook.go b/openfeature/flagevaluation_hook.go index 50019cf2d64..b24cc2af6d6 100644 --- a/openfeature/flagevaluation_hook.go +++ b/openfeature/flagevaluation_hook.go @@ -13,7 +13,7 @@ import ( // flagEvaluationHook implements the OpenFeature Hook interface to record EVP flagevaluation events. // It uses the Finally hook stage (same as flagEvalHook) to cover success, error, and default paths. -// This satisfies CONT-09: Finally fires on error/default paths, unlike After (PR #4874 defect). +// Finally fires on error/default paths, unlike After. type flagEvaluationHook struct { of.UnimplementedHook writer *flagEvaluationWriter @@ -25,8 +25,8 @@ func newFlagEvaluationHook(w *flagEvaluationWriter) *flagEvaluationHook { } // Finally is called after every flag evaluation (success or error). -// Using Finally (not After) ensures error-path and provider-not-ready evaluations are counted (CONT-09). -// Mirrors flageval_metrics.go's Finally stage; the EVP hook is a separate registered hook (D-06). +// Using Finally (not After) ensures error-path and provider-not-ready evaluations are counted. +// Mirrors flageval_metrics.go's Finally stage; the EVP hook is a separate registered hook. func (h *flagEvaluationHook) Finally( _ context.Context, hookContext of.HookContext, @@ -37,7 +37,7 @@ func (h *flagEvaluationHook) Finally( // frequently the request context, which may already be cancelled by the time // Finally runs — and record() is a non-blocking in-memory add with no network // call. Gating on ctx.Done() would silently drop legitimate evaluation counts - // (undercuts Gate-7 for cancelled-request evals) (WR-03). + // for cancelled-request evals. if h.writer == nil { return } @@ -47,7 +47,6 @@ func (h *flagEvaluationHook) Finally( // isRuntimeDefault returns true when the caller's supplied default value was returned. // Primary signal: absent variant key (flag-not-found, provider-not-ready, type-mismatch, no allocation). // Secondary: explicit DEFAULT or DISABLED reason (belt-and-suspenders). -// Satisfies CONT-07 (source_comment_id 3395344504). func isRuntimeDefault(details of.InterfaceEvaluationDetails) bool { if details.Variant == "" { return true diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index 7131d04b03f..00a5e3d48c2 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -22,6 +22,7 @@ func setupTestWriter(t *testing.T) *flagEvaluationWriter { flushInterval: 24 * time.Hour, // effectively disabled; tests control flush manually jsonConfig: jsoniter.Config{}.Froze(), stopChan: make(chan struct{}), + events: make(chan evalEvent, defaultEvalEventBufferSize), aggregator: flagEvaluationAggregator{ full: make(map[evaluationAggregationKey]*evaluationEntry), degraded: make(map[evaluationDegradedKey]*evaluationEntry), @@ -64,7 +65,7 @@ func makeEvalDetails(variant string, reason of.Reason, errorCode of.ErrorCode, m return d } -// TestIsRuntimeDefault verifies the runtime-default detection rule (CONT-07 / source_comment_id 3395344504). +// TestIsRuntimeDefault verifies the runtime-default detection rule. // Primary signal: variant=="" → true. // Secondary signal: reason==DEFAULT or reason==DISABLED → true even if variant is non-empty. // @@ -128,7 +129,7 @@ func TestIsRuntimeDefault(t *testing.T) { } // TestFlagEvaluationHookFinally verifies that the Finally hook records an entry for -// success, error-reason, and provider-not-ready paths (CONT-09 / source_comment_id 3385309423). +// success, error-reason, and provider-not-ready paths. // // It must fail RED: the hook's Finally method panics with "not implemented". func TestFlagEvaluationHookFinally(t *testing.T) { @@ -143,6 +144,10 @@ func TestFlagEvaluationHookFinally(t *testing.T) { hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + // record() enqueues asynchronously; drain the one event into the aggregator so the + // assertions below observe it deterministically (no worker runs in this test). + w.aggregate(<-w.events) + w.aggregator.mu.Lock() defer w.aggregator.mu.Unlock() @@ -161,6 +166,10 @@ func TestFlagEvaluationHookFinally(t *testing.T) { hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + // record() enqueues asynchronously; drain the one event into the aggregator so the + // assertions below observe it deterministically (no worker runs in this test). + w.aggregate(<-w.events) + w.aggregator.mu.Lock() defer w.aggregator.mu.Unlock() @@ -179,6 +188,10 @@ func TestFlagEvaluationHookFinally(t *testing.T) { hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + // record() enqueues asynchronously; drain the one event into the aggregator so the + // assertions below observe it deterministically (no worker runs in this test). + w.aggregate(<-w.events) + w.aggregator.mu.Lock() defer w.aggregator.mu.Unlock() @@ -197,6 +210,10 @@ func TestFlagEvaluationHookFinally(t *testing.T) { hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + // record() enqueues asynchronously; drain the one event into the aggregator so the + // assertions below observe it deterministically (no worker runs in this test). + w.aggregate(<-w.events) + w.aggregator.mu.Lock() defer w.aggregator.mu.Unlock() @@ -215,8 +232,8 @@ func TestFlagEvaluationHookFinally(t *testing.T) { } // TestFlagEvaluationHookContextCancelled verifies that a cancelled context does NOT -// drop the evaluation: record() is a non-blocking in-memory add, so a cancelled -// request context must still be counted (Gate-7 / WR-03). +// drop the evaluation: record() is a non-blocking enqueue that ignores the request +// context, so a cancelled request must still be counted. func TestFlagEvaluationHookContextCancelled(t *testing.T) { w := setupTestWriter(t) hook := newFlagEvaluationHook(w) @@ -229,6 +246,9 @@ func TestFlagEvaluationHookContextCancelled(t *testing.T) { hook.Finally(ctx, hookCtx, details, of.HookHints{}) + // record() enqueues asynchronously; drain the one event so the assertion sees it. + w.aggregate(<-w.events) + w.aggregator.mu.Lock() defer w.aggregator.mu.Unlock() @@ -237,3 +257,22 @@ func TestFlagEvaluationHookContextCancelled(t *testing.T) { t.Errorf("expected the cancelled-context evaluation to still be counted, got %d entries", total) } } + +// TestFlagEvaluationBackpressureDrops verifies the explicit backpressure policy: when the +// async hand-off queue is full, record() drops the event and counts it (observable) rather +// than blocking the evaluation. No worker drains in this test, so the queue fills after +// defaultEvalEventBufferSize enqueues and every further record() is a counted drop. +func TestFlagEvaluationBackpressureDrops(t *testing.T) { + w := setupTestWriter(t) + hookCtx := makeHookContext("bp-flag", "user-1", nil) + details := makeEvalDetails("on", of.TargetingMatchReason, "") + + const overflow = 100 + for range defaultEvalEventBufferSize + overflow { + w.record(hookCtx, details) // must never block, even once the queue is full + } + + if got := w.dropped.Load(); got != overflow { + t.Errorf("expected exactly %d dropped evaluations once the queue filled, got %d", overflow, got) + } +} diff --git a/openfeature/flagevaluation_provider_test.go b/openfeature/flagevaluation_provider_test.go index c37e065d137..541fd4cf243 100644 --- a/openfeature/flagevaluation_provider_test.go +++ b/openfeature/flagevaluation_provider_test.go @@ -13,7 +13,7 @@ import ( ) // TestFlagEvaluationKillswitch verifies that DD_FLAGGING_EVALUATION_COUNTS_ENABLED (default true) -// controls ONLY the EVP flagevaluation hook/writer, leaving the OTel flagEvalHook unaffected (PRES-01). +// controls ONLY the EVP flagevaluation hook/writer, leaving the OTel flagEvalHook unaffected. // // When the killswitch is "false": the EVP hook (flagEvalEVPHook) is NOT registered in Hooks() // and flagEvalWriter is nil. @@ -34,7 +34,7 @@ func TestFlagEvaluationKillswitch(t *testing.T) { hooks := p.Hooks() - // OTel hook must still be present (PRES-01). + // OTel hook must still be present. otelPresent := false for _, h := range hooks { if _, ok := h.(*flagEvalHook); ok { diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index b2bc6e4b876..966fdef1adc 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -30,7 +30,7 @@ func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { } // TestPruneContext verifies that pruneContext applies the 256-field / 256-char limits -// before evaluation context enters the aggregation buffer (D-07, CONT-03, T-DoS-mem). +// before evaluation context enters the aggregation buffer. // // It must fail RED: pruneContext panics with "not implemented". func TestPruneContext(t *testing.T) { @@ -95,7 +95,7 @@ func TestPruneContext(t *testing.T) { // TestFlagEvaluationPayloadSchema verifies that full, degraded, and ultra-degraded events // marshal to JSON that omits the expected optional fields per tier while always including -// the 5 required fields (CONT-04, CONT-02). +// the 5 required fields. // // It must fail RED: the aggregator.add method panics with "not implemented". func TestFlagEvaluationPayloadSchema(t *testing.T) { @@ -268,8 +268,7 @@ func TestFlagEvaluationPayloadSchema(t *testing.T) { } // TestAggregatorCollisionSafety verifies that two distinct inputs that would collide -// under FNV-1a-only map keying land in SEPARATE buckets under the struct-keyed map -// (CONT-05 / source_comment_id 3395004724). +// under FNV-1a-only map keying land in SEPARATE buckets under the struct-keyed map. // // It must fail RED: aggregator.add panics with "not implemented". func TestAggregatorCollisionSafety(t *testing.T) { @@ -313,7 +312,7 @@ func TestAggregatorCollisionSafety(t *testing.T) { } // TestAggregatorConcurrentMinMax verifies that 1000 goroutines recording the same key -// produce count==1000 and firstEvaluation<=lastEvaluation (CONT-06 / source_comment_id 3395176782). +// produce count==1000 and firstEvaluation<=lastEvaluation. // Must be run with -race to satisfy the race-free requirement. func TestAggregatorConcurrentMinMax(t *testing.T) { agg := &flagEvaluationAggregator{ @@ -363,14 +362,13 @@ func TestAggregatorConcurrentMinMax(t *testing.T) { } } -// TestSaturationCountPreservation is the regression guard for BLOCKER #1 (silent drop at -// globalCap overflow). It asserts that the sum of all evaluation counts across ALL tiers +// TestSaturationCountPreservation is the regression guard against a silent drop at +// globalCap overflow. It asserts that the sum of all evaluation counts across ALL tiers // (full + degraded + ultra-degraded) equals the total number of add() calls, even after // BOTH globalCap AND perFlagCap have been exhausted. // // This test MUST FAIL on the pre-fix code (negative control proving the silent drop), and -// MUST PASS after rerouting the globalCap-overflow return into the ultra-degraded tier -// (CONT-10 / D-08 / D-09 / source_comment_id 3385309423). +// MUST PASS after rerouting the globalCap-overflow return into the ultra-degraded tier. func TestSaturationCountPreservation(t *testing.T) { // Use small caps so we can saturate them quickly. // globalCap=5 means only 5 full-tier buckets ever created. @@ -395,8 +393,8 @@ func TestSaturationCountPreservation(t *testing.T) { // - The next ones overflow to degraded (bounded by degradedCap=3). // - Once degraded is also full, overflow to ultra-degraded. // - Once globalCap(5) is hit, any flag's attempt-count not yet at perFlagCap routes - // through the globalCap branch — the BLOCKER is that this branch returns silently - // instead of routing to ultra-degraded. + // through the globalCap branch — the defect being guarded against is that this branch + // returns silently instead of routing to ultra-degraded. const totalCalls = 100 for i := range totalCalls { flagIdx := i % 20 @@ -437,9 +435,9 @@ func TestSaturationCountPreservation(t *testing.T) { } // TestAggregatorCapOverflow verifies that: -// - Exceeding perFlagCap routes new entries to the degraded map (CONT-10). -// - Exceeding degradedCap routes new entries to the ultra-degraded map (CONT-10). -// - Global cap bounds total bucket growth (D-08 / source_comment_id 3385309427). +// - Exceeding perFlagCap routes new entries to the degraded map. +// - Exceeding degradedCap routes new entries to the ultra-degraded map. +// - Global cap bounds total bucket growth. func TestAggregatorCapOverflow(t *testing.T) { t.Run("perFlagCap overflow routes to degraded", func(t *testing.T) { agg := setupTestAggregator(t) // perFlagCap=3 @@ -536,7 +534,7 @@ func TestAggregatorCapOverflow(t *testing.T) { // globalCap=10 caps the full tier; overflow goes to ultra-degraded. // The full tier must stay at or below globalCap. // The total count across all tiers must equal the number of add() calls - // (no silent drops — CONT-10/D-08/D-09). + // (no silent drops). const calls = 50 for i := range calls { d := evalDetails{ diff --git a/openfeature/provider.go b/openfeature/provider.go index 9c4d3dcd252..de606652bee 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -36,7 +36,7 @@ const ( ffeProductEnvVar = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED" // flagEvalCountsEnabledEnvVar is the operator killswitch for the EVP flagevaluation emission path. // Default: true (EVP path is ON by default). Set to "false" to disable only the EVP path - // while leaving the OTel feature_flag.evaluations path unaffected (PRES-01). + // while leaving the OTel feature_flag.evaluations path unaffected. // Mirrors the internal.BoolEnv convention used by ffeProductEnvVar. flagEvalCountsEnabledEnvVar = "DD_FLAGGING_EVALUATION_COUNTS_ENABLED" // Default timeout for provider initialization @@ -52,7 +52,7 @@ type ProviderConfig struct { ExposureFlushInterval time.Duration // FlagEvaluationFlushInterval is the interval for flushing EVP flag evaluation events. - // Default: 10 seconds (D-05 contract value). Leave zero to use the default. + // Default: 10 seconds. Leave zero to use the default. FlagEvaluationFlushInterval time.Duration } @@ -122,7 +122,7 @@ func newDatadogProvider(config ProviderConfig) *DatadogProvider { // Conditionally construct the EVP flagevaluation writer + hook. // Gated by DD_FLAGGING_EVALUATION_COUNTS_ENABLED (default true). // When false, both fields are left nil and the EVP path is disabled. - // The OTel hook (flagEvalHook above) is registered unconditionally (PRES-01). + // The OTel hook (flagEvalHook above) is registered unconditionally. if internal.BoolEnv(flagEvalCountsEnabledEnvVar, true) { evalWriter := newFlagEvaluationWriter(config) p.flagEvalWriter = evalWriter @@ -451,7 +451,7 @@ func (p *DatadogProvider) Hooks() []openfeature.Hook { if p.exposureHook != nil { hooks = append(hooks, p.exposureHook) } - // OTel hook is always registered — untouched by the EVP killswitch (PRES-01). + // OTel hook is always registered — untouched by the EVP killswitch. if p.flagEvalHook != nil { hooks = append(hooks, p.flagEvalHook) } diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 231427bb875..6a8a825e865 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -7,6 +7,7 @@ package openfeature import ( "context" + "sort" "strconv" "testing" "time" @@ -153,7 +154,9 @@ func BenchmarkEvaluationWithVaryingContextSize(b *testing.B) { } } -// BenchmarkEvaluationWithVaryingFlagCounts benchmarks evaluation with different numbers of flags in config +// BenchmarkEvaluationWithVaryingFlagCounts benchmarks evaluation across configs with +// different numbers of flags. The evaluated flag key rotates across all configured flags, +// so flag-cardinality is exercised rather than repeatedly evaluating a single key. func BenchmarkEvaluationWithVaryingFlagCounts(b *testing.B) { flagCounts := []struct { name string @@ -203,11 +206,17 @@ func BenchmarkEvaluationWithVaryingFlagCounts(b *testing.B) { "country": "US", } + // Rotate across all boolean flags so a larger config is actually exercised, + // not just a single repeated key. + flagKeys := boolBenchmarkFlagKeys(config) + b.ReportAllocs() b.ResetTimer() + i := 0 for b.Loop() { - _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) + _ = provider.BooleanEvaluation(ctx, flagKeys[i%len(flagKeys)], false, flatCtx) + i++ } }) } @@ -238,7 +247,7 @@ func BenchmarkConcurrentEvaluations(b *testing.B) { // makeBenchmarkConfig creates a test config with the specified number of flags. // Extends createTestConfig() for benchmark load profiles. // Flag keys are unique monotonic integers ("bench-flag-N") so the claimed cardinality -// is fully exercised without wrapping (WR-04). +// is fully exercised without wrapping. func makeBenchmarkConfig(numFlags int) *universalFlagsConfiguration { config := createTestConfig() for i := len(config.Flags); i < numFlags; i++ { @@ -264,146 +273,159 @@ func makeBenchmarkConfig(numFlags int) *universalFlagsConfiguration { return config } -// makeBenchmarkContext creates a FlattenedContext with numFields attributes. -// Field names are "fieldN" with a monotonic index so all numFields are distinct and -// the claimed context cardinality is fully exercised without wrapping (WR-04). -func makeBenchmarkContext(numFields int) openfeature.FlattenedContext { - ctx := openfeature.FlattenedContext{ - "targetingKey": "bench-user-001", +// boolBenchmarkFlagKeys returns the boolean flag keys in cfg in deterministic order, so a +// benchmark can rotate the evaluated flag and exercise flag-cardinality (not just users). +func boolBenchmarkFlagKeys(cfg *universalFlagsConfiguration) []string { + keys := make([]string, 0, len(cfg.Flags)) + for k, f := range cfg.Flags { + if f.VariationType == valueTypeBoolean { + keys = append(keys, k) + } } + sort.Strings(keys) + return keys +} + +// makeBenchmarkAttrs builds an evaluation-context attribute map with numFields-1 distinct +// fields (the targeting key is supplied separately to NewEvaluationContext, so the flattened +// context has numFields entries total). Field names are "fieldN" so all are distinct. +func makeBenchmarkAttrs(numFields int) map[string]any { + attrs := make(map[string]any, numFields) for i := 1; i < numFields; i++ { - ctx["field"+strconv.Itoa(i)] = "value" + attrs["field"+strconv.Itoa(i)] = "value" } - return ctx + return attrs } -// BenchmarkFlagEvaluationNoop measures the raw evaluation cost with zero hooks — -// the pure evaluator baseline for the three-column overhead comparison (CONT-08 / D-11). -// -// Profile: typical (100 flags, 50-user simulation, 10-field context). -// Profile: stress (10 flags, 1000-user simulation, 250-field context — near degraded trigger). -// -// Run command: -// -// GOFLAGS=-mod=readonly go test ./openfeature -run='^$' -bench='^BenchmarkFlagEvaluation' \ -// -benchmem -count=3 -cpu=8 -func BenchmarkFlagEvaluationNoop(b *testing.B) { - profiles := []struct { - name string - numFlags int - numUsers int - numFields int - }{ - {"typical/100flags_50users_10fields", 100, 50, 10}, - {"stress/10flags_1000users_250fields", 10, 1000, 250}, - } +// flagEvalBenchProfile is one load profile for the three-column overhead comparison. +type flagEvalBenchProfile struct { + name string + numFlags int + numUsers int + numFields int +} - for _, p := range profiles { +// flagEvalBenchProfiles are shared by the Noop / OTel-only / OTel+EVP benchmarks so the +// three columns are directly comparable. +var flagEvalBenchProfiles = []flagEvalBenchProfile{ + {"typical/100flags_50users_10fields", 100, 50, 10}, + {"stress/10flags_1000users_250fields", 10, 1000, 250}, +} + +// runFlagEvalBenchmark drives evaluations through a real OpenFeature client so the +// provider's registered hooks actually run. Calling provider.BooleanEvaluation directly +// would bypass Hooks(), so neither the OTel nor the EVP hook would execute and all three +// columns would measure the same bare evaluator. Both the evaluated flag key and the +// targeting key rotate, so flag- and user-cardinality are exercised. +// +// configureHooks nils whichever provider hooks the tier under test should exclude. It runs +// before the provider is registered; Init does not recreate hooks, so the choice sticks. +// The provider uses a 24h flush interval, so the EVP writer never flushes during the run — +// no HTTP round-trip enters the hot path, isolating hook + aggregator cost. +func runFlagEvalBenchmark(b *testing.B, configureHooks func(p *DatadogProvider)) { + b.Helper() + for _, p := range flagEvalBenchProfiles { b.Run(p.name, func(b *testing.B) { - // Noop: provider with no hooks (nil out hooks after construction) - provider := newDatadogProvider(ProviderConfig{}) - provider.flagEvalHook = nil // no OTel hook - provider.exposureHook = nil // no exposure hook + provider := newDatadogProvider(ProviderConfig{FlagEvaluationFlushInterval: 24 * time.Hour}) + configureHooks(provider) + config := makeBenchmarkConfig(p.numFlags) provider.updateConfiguration(config) + initCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := openfeature.SetProviderWithContextAndWait(initCtx, provider); err != nil { + cancel() + b.Fatalf("set provider: %v", err) + } + cancel() + client := openfeature.NewClient("flageval-bench") + + flagKeys := boolBenchmarkFlagKeys(config) + attrs := makeBenchmarkAttrs(p.numFields) ctx := context.Background() - flatCtx := makeBenchmarkContext(p.numFields) b.ReportAllocs() b.ResetTimer() i := 0 for b.Loop() { - // Rotate user targeting key to simulate p.numUsers distinct users. - // Use a per-iteration counter (not b.N, which is a single fixed total - // under b.Loop()) so the cardinality profile is actually exercised (WR-04). - flatCtx["targetingKey"] = "bench-user-" + strconv.Itoa(i%p.numUsers) + evalCtx := openfeature.NewEvaluationContext("bench-user-"+strconv.Itoa(i%p.numUsers), attrs) + _ = client.Boolean(ctx, flagKeys[i%len(flagKeys)], false, evalCtx) i++ - _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) } }) } } -// BenchmarkFlagEvaluationOTelOnly measures the evaluation cost with only the existing -// OTel feature_flag.evaluations hook (Path A — preserved baseline) (CONT-08 / D-11). -func BenchmarkFlagEvaluationOTelOnly(b *testing.B) { - profiles := []struct { - name string - numFlags int - numUsers int - numFields int - }{ - {"typical/100flags_50users_10fields", 100, 50, 10}, - {"stress/10flags_1000users_250fields", 10, 1000, 250}, - } - - for _, p := range profiles { - b.Run(p.name, func(b *testing.B) { - // OTel-only: provider with flagEvalHook (OTel) but no EVP hook - provider := newDatadogProvider(ProviderConfig{}) - provider.exposureHook = nil // no exposure hook — isolate OTel cost only - // provider.flagEvalHook is set by newDatadogProvider — keep it - config := makeBenchmarkConfig(p.numFlags) - provider.updateConfiguration(config) - - ctx := context.Background() - flatCtx := makeBenchmarkContext(p.numFields) - - b.ReportAllocs() - b.ResetTimer() +// BenchmarkFlagEvaluationNoop measures the client+provider evaluation cost with no hooks — +// the baseline column for the three-column overhead comparison. +// +// Run command: +// +// GOFLAGS=-mod=readonly go test ./openfeature -run='^$' -bench='^BenchmarkFlagEvaluation' \ +// -benchmem -count=3 -cpu=8 +func BenchmarkFlagEvaluationNoop(b *testing.B) { + runFlagEvalBenchmark(b, func(p *DatadogProvider) { + p.flagEvalHook = nil + p.flagEvalEVPHook = nil + p.flagEvalWriter = nil + p.exposureHook = nil + }) +} - i := 0 - for b.Loop() { - flatCtx["targetingKey"] = "bench-user-" + strconv.Itoa(i%p.numUsers) - i++ - _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) - } - }) - } +// BenchmarkFlagEvaluationOTelOnly measures the cost with only the existing OTel +// feature_flag.evaluations hook (Path A — the preserved baseline). The EVP hook and writer +// are disabled so this column isolates the OTel hook's cost. +func BenchmarkFlagEvaluationOTelOnly(b *testing.B) { + runFlagEvalBenchmark(b, func(p *DatadogProvider) { + p.flagEvalEVPHook = nil + p.flagEvalWriter = nil + p.exposureHook = nil + }) } // BenchmarkFlagEvaluationOTelPlusEVP measures the marginal cost of adding the new EVP -// flagevaluation hook alongside the existing OTel hook (Path A + Path B) (CONT-08 / D-11). -// -// The EVP writer uses a 24-hour flush interval (effectively infinite) so no HTTP round-trip -// is in the hot path — this benchmarks only the hook + aggregator cost (RESEARCH Pitfall 4). +// flagevaluation hook alongside the existing OTel hook (Path A + Path B). Only the exposure +// hook is disabled; the OTel hook, EVP hook, and aggregator all run. func BenchmarkFlagEvaluationOTelPlusEVP(b *testing.B) { - profiles := []struct { - name string - numFlags int - numUsers int - numFields int - }{ - {"typical/100flags_50users_10fields", 100, 50, 10}, - {"stress/10flags_1000users_250fields", 10, 1000, 250}, - } + runFlagEvalBenchmark(b, func(p *DatadogProvider) { + p.exposureHook = nil + }) +} - for _, p := range profiles { +// BenchmarkFlagEvaluationEVPRecord isolates the EVP hook's synchronous hot-path cost — the +// scalar extraction + shallow context copy (EvaluationContext().Attributes()) + non-blocking +// enqueue that runs on the evaluation goroutine. A discard drainer empties the queue so it +// never fills, and the asynchronous aggregation (flatten/prune/hash/add, performed by the +// real worker off the hot path) is NOT attributed here. This is the latency a flag +// evaluation actually pays when the EVP path is enabled — distinct from the client +// benchmarks above, whose process-wide allocs/wall-clock also capture the worker's work. +func BenchmarkFlagEvaluationEVPRecord(b *testing.B) { + for _, p := range flagEvalBenchProfiles { b.Run(p.name, func(b *testing.B) { - // OTel+EVP: provider with both OTel hook and EVP hook wired. - // The EVP writer uses a 24-hour flush interval so no HTTP round-trip - // is in the hot path — benchmarks hook + aggregator cost only. - provider := newDatadogProvider(ProviderConfig{ - FlagEvaluationFlushInterval: 24 * time.Hour, // never flush during bench - }) - provider.exposureHook = nil // isolate hook overhead; no exposure cost - - config := makeBenchmarkConfig(p.numFlags) - provider.updateConfiguration(config) + w := newFlagEvaluationWriter(ProviderConfig{FlagEvaluationFlushInterval: 24 * time.Hour}) + + // Discard drainer: keep the queue empty without aggregating, so only the + // synchronous enqueue cost is measured here. + done := make(chan struct{}) + go func() { + for { + select { + case <-w.events: + case <-done: + return + } + } + }() + defer close(done) - ctx := context.Background() - flatCtx := makeBenchmarkContext(p.numFields) + hookCtx := makeHookContext("bool-flag", "bench-user", makeBenchmarkAttrs(p.numFields)) + details := makeEvalDetails("on", openfeature.TargetingMatchReason, "") b.ReportAllocs() b.ResetTimer() - - i := 0 for b.Loop() { - flatCtx["targetingKey"] = "bench-user-" + strconv.Itoa(i%p.numUsers) - i++ - _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) + w.record(hookCtx, details) } }) } From 140d657f2fb0c1a5abfb6790cb97cf8e7d633bcb Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 08:52:00 -0400 Subject: [PATCH 13/37] fix(openfeature): correct EVP flagevaluation context hashing, ultra-degraded bound, post-stop record, and error message - Deterministic context pruning: sort keys BEFORE the 256-field cap and oversized-string skip so identical logical contexts prune to the same subset and hash equal (no bucket fragmentation). - Canonical context hash encoding: type-tagged + length-delimited key/value so int 1 != string "1" and '=' / '\n' in keys/values cannot alias a multi-field context. - Bounded ultra-degraded tier: explicit ultraDegCap (<=0 treated as default) with a single count-preserving sentinel overflow bucket (__other__) so dynamic/malicious flag keys cannot grow the map without bound. - Post-stop record() no-op: stopped is now atomic.Bool used as the single stop() idempotency gate; record() checks it lock-free and counts post-stop events as dropped instead of enqueuing into the never-drained channel. - Error payload prefers OpenFeature ErrorMessage, falling back to ErrorCode only when ErrorMessage is empty. - Doc comments on evaluationAggregationKey, hashContext, and addToUltraDegraded updated to match the fixed behavior. OTel feature_flag.evaluations path (flageval_metrics.go) is unchanged. --- openfeature/flagevaluation.go | 210 ++++++++++++++++++----- openfeature/flagevaluation_test.go | 265 +++++++++++++++++++++++++++++ 2 files changed, 428 insertions(+), 47 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 9a0aed05bb9..e798328d12f 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -51,6 +51,17 @@ const ( defaultEvalPerFlagCap = 10_000 // bounds full-fidelity buckets per flag defaultEvalDegradedCap = 10_000 // bounds degraded map + // defaultEvalUltraDegradedCap bounds the number of distinct ultra-degraded buckets. + // The ultra-degraded key is (flagKey, variant) and the flag key is caller-supplied, so a + // dynamic/malicious flag key would otherwise grow this map without bound. Distinct keys + // beyond the cap collapse into a single count-preserving sentinel overflow bucket. + defaultEvalUltraDegradedCap = 10_000 + + // ultraDegradedOverflowFlagKey is the reserved flag key for the single sentinel bucket + // that absorbs counts for all over-cap distinct ultra-degraded keys, so the total + // evaluation count is preserved even when the ultra-degraded cardinality is capped. + ultraDegradedOverflowFlagKey = "__other__" + // defaultEvalEventBufferSize bounds the async hand-off queue between the (hot-path) // Finally hook and the background aggregation worker. On overflow the hook drops the // event and increments a counter rather than blocking the evaluation. @@ -66,16 +77,21 @@ const ( // count can never be attributed to the wrong flag/variant/reason/allocation. // // 2. contextHash is a 64-bit FNV-1a digest of the pruned evaluation context (see -// hashContext, hashed over sorted keys for determinism). The context is map[string]any -// and therefore not comparable, so it cannot be a struct field — the digest stands in -// for it. The digest is deterministic and low-collision but NOT collision-proof. +// hashContext). The context is map[string]any and therefore not comparable, so it +// cannot be a struct field — the digest stands in for it. The digest is computed over a +// CANONICAL encoding: keys are sorted (deterministic ordering), and each field is +// written as a length-delimited key plus a type-tagged, length-delimited value. Distinct +// contexts therefore cannot ALIAS by construction — int 1 vs string "1" carry different +// type tags, and '=' / '\n' inside a key or value cannot fake a field boundary. // // What a contextHash collision actually does — and why it is acceptable: // -// A collision requires two evaluations that are identical on ALL FIVE exact dimensions -// AND whose *different* contexts happen to hash to the same uint64. When that occurs, the -// second evaluation matches the existing bucket on add()'s fast path and increments its -// count (the `if e, ok := a.full[fullKey]; ok` branch below). Consequences: +// With the canonical encoding above, a collision is no longer a systematic encoding alias; +// it is a genuine 64-bit FNV-1a hash collision: two evaluations identical on ALL FIVE +// exact dimensions AND whose *different* canonical encodings happen to hash to the same +// uint64. When that occurs, the second evaluation matches the existing bucket on add()'s +// fast path and increments its count (the `if e, ok := a.full[fullKey]; ok` branch below). +// Consequences: // // - COUNT-PRESERVING: the evaluation count is merged into the existing bucket, never // dropped. The invariant Σ(counts across all tiers) == number of add() calls still @@ -87,12 +103,9 @@ const ( // Context is a best-effort dimension that the degraded/ultra tiers drop entirely by // design, so a collision is strictly less lossy than ordinary degradation. // -// Probability is bounded by the cap: the full tier holds at most globalCap (65536) -// buckets, so there are <= ~65k distinct digests per flush window. Birthday bound -// ~= 65536^2 / 2^65 ~= 1.2e-10 per window. This is internal telemetry over the customer's -// own context — not a trust boundary — so a keyed/crypto hash (e.g. SipHash) is -// unwarranted, and a wider (128-bit) digest would buy only context-label fidelity at that -// 1e-10 tail. Deliberately not done. +// This is internal telemetry over the customer's own context — not a trust boundary — so a +// keyed/crypto hash (e.g. SipHash) is unwarranted, and a wider (128-bit) digest would buy +// only marginal context-label fidelity. Deliberately not done. type evaluationAggregationKey struct { flagKey string variant string @@ -112,7 +125,9 @@ type evaluationDegradedKey struct { } // evaluationUltraDegradedKey is the key for the ultra-degraded aggregation map. -// Contains only flag key + variant; bounded by flag×variant enumeration. +// Contains only flag key + variant. Because flagKey is caller-supplied, the distinct +// cardinality of this map is bounded EXPLICITLY by ultraDegCap (see addToUltraDegraded); +// over-cap distinct keys collapse into a single count-preserving sentinel bucket. type evaluationUltraDegradedKey struct { flagKey string variant string @@ -141,6 +156,7 @@ type flagEvaluationAggregator struct { globalCap int perFlagCap int degradedCap int + ultraDegCap int // bounds distinct ultra-degraded buckets; <=0 is treated as the default } // flagEvaluationEvent matches flagevaluation.json — required fields always present; @@ -212,7 +228,7 @@ type flagEvaluationWriter struct { ddContext flagEvalDDContext // service/env/version — same source as exposureContext ticker *time.Ticker stopChan chan struct{} - stopped bool + stopped atomic.Bool // single idempotency gate for stop(); also read lock-free by record() jsonConfig jsoniter.API // Asynchronous hand-off: the Finally hook enqueues a cheap snapshot here; a single @@ -283,6 +299,7 @@ func newFlagEvaluationWriter(config ProviderConfig) *flagEvaluationWriter { globalCap: defaultEvalGlobalCap, perFlagCap: defaultEvalPerFlagCap, degradedCap: defaultEvalDegradedCap, + ultraDegCap: defaultEvalUltraDegradedCap, }, } } @@ -325,13 +342,13 @@ func (w *flagEvaluationWriter) start() { // stop stops the flush ticker and marks the writer as stopped. func (w *flagEvaluationWriter) stop() { - w.aggregator.mu.Lock() - if w.stopped { - w.aggregator.mu.Unlock() + // Single idempotency gate: the atomic Swap is the ONLY guard for both "mark stopped" and + // the downstream close(stopChan). A second/concurrent stop() sees Swap return true and + // returns early, so stopChan is closed exactly once (no double-close panic). record() reads + // this flag lock-free to no-op post-stop enqueues. + if w.stopped.Swap(true) { return } - w.stopped = true - w.aggregator.mu.Unlock() // Signal the worker to drain the queue and do a final flush. close(w.stopChan) @@ -458,6 +475,13 @@ func (w *flagEvaluationWriter) flush() { // queue is full the event is dropped and counted (best-effort), never blocking the // evaluation. Called from the Finally hook after every evaluation. func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.InterfaceEvaluationDetails) { + // Post-stop no-op: after stop() the worker no longer drains w.events, so enqueuing would + // silently lose the event. Check the atomic gate lock-free (reading under the aggregator + // lock would add hot-path contention) and count the event as dropped so it stays observable. + if w.stopped.Load() { + w.dropped.Add(1) + return + } ev := evalEvent{ d: extractEvalDetails(hookContext, details), attrs: hookContext.EvaluationContext().Attributes(), // SDK returns an owned copy @@ -647,8 +671,14 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { // addToUltraDegraded adds an entry to the ultra-degraded map (only flag key + variant). // Called with the aggregator lock held. -// Ultra-degraded has no explicit cap — it is naturally bounded by the flag×variant -// enumeration. The globalCap enforcement applies only to the full tier. +// +// The ultra-degraded tier is EXPLICITLY capped by ultraDegCap (with <=0 treated as +// defaultEvalUltraDegradedCap). Because flagKey is caller-supplied, an unbounded number of +// dynamic/malicious flag keys would otherwise grow this map without bound (reviewer concern +// #8). When creating a NEW bucket would exceed the effective cap, the count is routed into a +// single count-preserving sentinel overflow bucket (flag key ultraDegradedOverflowFlagKey, +// empty variant) instead. Existing buckets always increment normally, so the total +// evaluation count is preserved (Σ across tiers == add() calls). func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64) { ultraKey := evaluationUltraDegradedKey{ flagKey: d.flagKey, @@ -666,7 +696,35 @@ func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64 return } - // New ultra-degraded bucket — always created (naturally bounded by flag×variant). + // New ultra-degraded bucket — enforce the explicit cap. Resolve the effective cap with + // the zero-value footgun policy: a non-positive ultraDegCap means "use the default" so a + // forgotten field does not silently route everything to the sentinel. + cap := a.ultraDegCap + if cap <= 0 { + cap = defaultEvalUltraDegradedCap + } + if len(a.ultraDeg) >= cap { + // Cap reached — collapse this and all further over-cap distinct keys into a single + // sentinel overflow bucket so the count is preserved without unbounded growth. + overflowKey := evaluationUltraDegradedKey{flagKey: ultraDegradedOverflowFlagKey} + if e, ok := a.ultraDeg[overflowKey]; ok { + e.count++ + if nowMs < e.firstEvaluation { + e.firstEvaluation = nowMs + } + if nowMs > e.lastEvaluation { + e.lastEvaluation = nowMs + } + return + } + a.ultraDeg[overflowKey] = &evaluationEntry{ + count: 1, + firstEvaluation: nowMs, + lastEvaluation: nowMs, + } + return + } + a.ultraDeg[ultraKey] = &evaluationEntry{ count: 1, firstEvaluation: nowMs, @@ -674,15 +732,33 @@ func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64 } } +// context value type discriminators for the canonical hash encoding. Each distinct Go type +// gets a distinct tag byte so that, e.g., int 1 and string "1" cannot render identically. +const ( + ctxTagString byte = 's' + ctxTagBool byte = 'b' + ctxTagInt byte = 'i' + ctxTagInt64 byte = 'l' + ctxTagInt32 byte = 'j' + ctxTagFloat64 byte = 'f' + ctxTagFloat32 byte = 'g' + ctxTagOther byte = 'o' +) + // hashContext computes a deterministic 64-bit FNV-1a hash of the pruned context map for // use as a discriminator inside evaluationAggregationKey. The enumerable struct fields // (flagKey, variant, allocationKey, reason, targetingKey) are exact string comparisons and -// cannot collide; contextHash is a probabilistic supplement for context identity only — -// it is low-collision but NOT collision-proof. +// cannot collide. // -// A digest collision is count-preserving (it merges into an existing bucket via add()'s -// fast path, costing only context-attribute fidelity, never a count) — see the -// evaluationAggregationKey doc for the full collision analysis. +// The per-field encoding is CANONICAL: each field is written as a length-delimited key +// (fixed-width 8-byte big-endian length + key bytes) followed by a type-tag byte and a +// length-delimited rendered value. Because both key and value are length-delimited and the +// value is type-tagged, distinct contexts cannot ALIAS by construction: int 1 and string +// "1" carry different tags, and '=' / '\n' embedded in a key or value can no longer be +// mistaken for a field delimiter. A digest collision is therefore now a genuine 64-bit hash +// collision, not a systematic encoding alias. A collision is still count-preserving (it +// merges into an existing bucket via add()'s fast path, costing only context-attribute +// fidelity, never a count) — see the evaluationAggregationKey doc for the full analysis. func hashContext(attrs map[string]any) uint64 { if len(attrs) == 0 { return 0 @@ -703,55 +779,93 @@ func hashContext(attrs map[string]any) uint64 { var buf []byte for _, k := range keys { buf = buf[:0] - buf = append(buf, k...) - buf = append(buf, '=') - buf = appendContextValue(buf, attrs[k]) - buf = append(buf, '\n') + buf = appendLengthDelimited(buf, []byte(k)) // length-delimited key + buf = appendContextValue(buf, attrs[k]) // tag + length-delimited value _, _ = h.Write(buf) } return h.Sum64() } -// appendContextValue appends a deterministic string form of v to buf, avoiding allocation -// for the common scalar types; rare/complex types fall back to fmt. The hash is only an -// in-memory, per-flush-window discriminator, so the exact formatting is irrelevant as long -// as it is deterministic within a run. +// appendLengthDelimited writes a fixed-width 8-byte big-endian length prefix followed by the +// raw bytes, so the boundary between fields is unambiguous regardless of the byte content. +func appendLengthDelimited(buf, b []byte) []byte { + var lenBuf [8]byte + n := uint64(len(b)) + for i := range 8 { + lenBuf[7-i] = byte(n) + n >>= 8 + } + buf = append(buf, lenBuf[:]...) + return append(buf, b...) +} + +// appendContextValue appends a CANONICAL, length-delimited rendering of v to buf: a type-tag +// byte (distinct per Go type) followed by a length-delimited rendered value. This avoids +// allocation for the common scalar types; rare/complex types fall back to fmt. The encoding +// only needs to be deterministic within a run and collision-free across distinct values. func appendContextValue(buf []byte, v any) []byte { + var scratch [32]byte + tmp := scratch[:0] + var tag byte switch x := v.(type) { case string: - return append(buf, x...) + tag = ctxTagString + tmp = append(tmp, x...) case bool: - return strconv.AppendBool(buf, x) + tag = ctxTagBool + tmp = strconv.AppendBool(tmp, x) case int: - return strconv.AppendInt(buf, int64(x), 10) + tag = ctxTagInt + tmp = strconv.AppendInt(tmp, int64(x), 10) case int64: - return strconv.AppendInt(buf, x, 10) + tag = ctxTagInt64 + tmp = strconv.AppendInt(tmp, x, 10) case int32: - return strconv.AppendInt(buf, int64(x), 10) + tag = ctxTagInt32 + tmp = strconv.AppendInt(tmp, int64(x), 10) case float64: - return strconv.AppendFloat(buf, x, 'g', -1, 64) + tag = ctxTagFloat64 + tmp = strconv.AppendFloat(tmp, x, 'g', -1, 64) case float32: - return strconv.AppendFloat(buf, float64(x), 'g', -1, 32) + tag = ctxTagFloat32 + tmp = strconv.AppendFloat(tmp, float64(x), 'g', -1, 32) default: - return append(buf, fmt.Sprintf("%v", x)...) + tag = ctxTagOther + tmp = append(tmp, fmt.Sprintf("%v", x)...) } + buf = append(buf, tag) + return appendLengthDelimited(buf, tmp) } // pruneContext applies 256-field / 256-char limits before buffering. // Mirrors worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH exactly. // Must be called AFTER flattenContext() (from flatten.go) to expand nested objects first. +// +// The kept subset is DETERMINISTIC: keys are sorted BEFORE the oversized-string skip and the +// 256-field cap are applied, so two logically-identical contexts always prune to the exact +// same subset (and therefore the same hashContext digest). Ranging the map directly (Go map +// iteration is randomized) would cut a different 256-field subset each call and fragment +// otherwise-identical contexts into separate aggregation buckets. func pruneContext(raw map[string]any) map[string]any { if len(raw) == 0 { return nil } + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + out := make(map[string]any, min(len(raw), maxContextFields)) count := 0 - for k, v := range raw { + for _, k := range keys { if count >= maxContextFields { break } + v := raw[k] if s, ok := v.(string); ok && len(s) > maxFieldLength { // Skip oversized string values (matches worker.ts pruneFields behavior). + // Applied against the deterministic ordering so the kept subset is stable. continue } out[k] = v @@ -772,8 +886,10 @@ func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluati if reason == "" { reason = "unknown" } - var errMsg string - if details.ErrorCode != "" { + // Prefer OpenFeature's human-readable ErrorMessage; fall back to the ErrorCode string only + // when ErrorMessage is empty (some providers populate just the code). + errMsg := details.ErrorMessage + if errMsg == "" && details.ErrorCode != "" { errMsg = string(details.ErrorCode) } return evalDetails{ diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 966fdef1adc..141543c1418 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -12,6 +12,8 @@ import ( "sync" "testing" "time" + + of "github.com/open-feature/go-sdk/openfeature" ) // setupTestAggregator creates a flagEvaluationAggregator with small caps for testing. @@ -26,6 +28,7 @@ func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { globalCap: 10, // small cap to trigger overflow in tests perFlagCap: 3, degradedCap: 3, + ultraDegCap: defaultEvalUltraDegradedCap, // real default keeps the multi-tier cascade from collapsing into the sentinel } } @@ -323,6 +326,7 @@ func TestAggregatorConcurrentMinMax(t *testing.T) { globalCap: 100_000, // large enough not to overflow during this test perFlagCap: 100_000, degradedCap: 100_000, + ultraDegCap: defaultEvalUltraDegradedCap, } d := evalDetails{ @@ -382,6 +386,7 @@ func TestSaturationCountPreservation(t *testing.T) { globalCap: 5, perFlagCap: 2, degradedCap: 3, + ultraDegCap: defaultEvalUltraDegradedCap, } nowMs := time.Now().UnixMilli() @@ -572,3 +577,263 @@ func TestAggregatorCapOverflow(t *testing.T) { } }) } + +// TestPruneContextDeterministic guards Finding #1 (nondeterministic context pruning). +// A >256-field context must prune to an IDENTICAL kept subset (and therefore an identical +// hash) on every call, and two independently-built maps with the same logical entries must +// hash equal. On the pre-fix code (map-range cap BEFORE sort) the kept subset is random, so +// the hash varies across iterations and identical logical contexts fragment into separate +// buckets. +func TestPruneContextDeterministic(t *testing.T) { + const fields = 400 // > maxContextFields (256) + build := func() map[string]any { + m := make(map[string]any, fields) + for i := range fields { + m[fmt.Sprintf("key%04d", i)] = fmt.Sprintf("value%04d", i) + } + return m + } + + first := hashContext(pruneContext(build())) + for range 50 { + got := hashContext(pruneContext(build())) + if got != first { + t.Fatalf("pruneContext+hashContext nondeterministic over >256 fields: got %d, want %d", got, first) + } + } + + // Two independently-built maps with the SAME 400 logical entries must hash equal. + if a, b := hashContext(pruneContext(build())), hashContext(pruneContext(build())); a != b { + t.Errorf("identical logical contexts hashed differently: %d != %d", a, b) + } +} + +// TestPruneContextOversizedStringDeterministic guards Finding #1 with an oversized-string +// skip among >256 fields: the oversized-string skip must be applied against a deterministic +// key ordering, so the kept subset (and hash) is stable across iterations. +func TestPruneContextOversizedStringDeterministic(t *testing.T) { + const fields = 400 + longVal := strings.Repeat("x", maxFieldLength+44) // > maxFieldLength → skipped + build := func() map[string]any { + m := make(map[string]any, fields) + for i := range fields { + m[fmt.Sprintf("key%04d", i)] = fmt.Sprintf("value%04d", i) + } + m["zzz-oversized"] = longVal // sorts last; deterministically skipped + return m + } + + first := hashContext(pruneContext(build())) + for range 50 { + got := hashContext(pruneContext(build())) + if got != first { + t.Fatalf("oversized-string prune nondeterministic: got %d, want %d", got, first) + } + } + + // The oversized value must never appear in the pruned subset. + pruned := pruneContext(build()) + if _, ok := pruned["zzz-oversized"]; ok { + t.Error("oversized string value should be skipped from pruned context") + } +} + +// TestHashContextCanonicalEncoding guards Finding #2 (non-canonical context hash encoding). +// Distinct contexts must not alias: int 1 vs string "1" must differ, and '='/'\n'-bearing +// values/keys must not be able to fake a multi-field context. Drives a subset through +// aggregator.add and asserts SEPARATE full-tier buckets. +func TestHashContextCanonicalEncoding(t *testing.T) { + t.Run("int 1 != string 1", func(t *testing.T) { + if hashContext(map[string]any{"x": 1}) == hashContext(map[string]any{"x": "1"}) { + t.Error("type-tagged encoding must distinguish int 1 from string \"1\"") + } + }) + + t.Run("'=' in key/value cannot alias a multi-field context", func(t *testing.T) { + // {"a=b":"c"} vs {"a":"b=c"} render identically under key+"="+value with no + // length delimiter; canonical encoding must keep them distinct. + if hashContext(map[string]any{"a=b": "c"}) == hashContext(map[string]any{"a": "b=c"}) { + t.Error("'=' in key/value must not alias across fields") + } + }) + + t.Run("'\\n' in value cannot alias a multi-field context", func(t *testing.T) { + // {"a":"x\ny":"z"} would, under key+"="+value+"\n", collide with a two-field map. + if hashContext(map[string]any{"a": "x\ny", "b": "z"}) == hashContext(map[string]any{"a": "x", "y=z": ""}) { + t.Error("'\\n' in value must not alias a multi-field context") + } + }) + + t.Run("distinct contexts land in separate full-tier buckets via add()", func(t *testing.T) { + agg := setupTestAggregator(t) + nowMs := time.Now().UnixMilli() + d := evalDetails{flagKey: "f", variant: "on", reason: "targeting_match"} + agg.add(d, map[string]any{"x": 1}, nowMs) + agg.add(d, map[string]any{"x": "1"}, nowMs) + + agg.mu.Lock() + defer agg.mu.Unlock() + if len(agg.full) != 2 { + t.Errorf("expected 2 full-tier buckets for int 1 vs string \"1\", got %d", len(agg.full)) + } + }) +} + +// TestUltraDegradedCapBounded guards Finding #3 (ultra-degraded tier unbounded). With a +// small positive ultraDegCap, many distinct flag keys must NOT grow ultraDeg without bound: +// len(ultraDeg) <= ultraDegCap+1 (the +1 is the sentinel overflow bucket) and counts are +// preserved (Σ across tiers == add() calls). +func TestUltraDegradedCapBounded(t *testing.T) { + const cap = 3 + agg := &flagEvaluationAggregator{ + full: make(map[evaluationAggregationKey]*evaluationEntry), + degraded: make(map[evaluationDegradedKey]*evaluationEntry), + ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), + perFlagFull: make(map[string]int), + globalCap: 0, // force everything straight into the ultra-degraded tier + perFlagCap: 100_000, + degradedCap: 0, + ultraDegCap: cap, + } + nowMs := time.Now().UnixMilli() + + const calls = 100 + for i := range calls { + d := evalDetails{ + flagKey: fmt.Sprintf("dynamic-flag-%d", i), // every key distinct + variant: "on", + reason: "targeting_match", + } + agg.add(d, nil, nowMs) + } + + agg.mu.Lock() + defer agg.mu.Unlock() + + if len(agg.ultraDeg) > cap+1 { + t.Errorf("ultra-degraded cardinality %d exceeds ultraDegCap+1 (%d) — tier not bounded", len(agg.ultraDeg), cap+1) + } + + var total int64 + for _, e := range agg.full { + total += e.count + } + for _, e := range agg.degraded { + total += e.count + } + for _, e := range agg.ultraDeg { + total += e.count + } + if total != calls { + t.Errorf("count preservation violated under ultraDegCap: Σ(all tiers)=%d, expected=%d", total, calls) + } + + // The sentinel overflow bucket must be present and carry the over-cap counts. + if _, ok := agg.ultraDeg[evaluationUltraDegradedKey{flagKey: ultraDegradedOverflowFlagKey}]; !ok { + t.Errorf("expected sentinel overflow bucket %q in ultra-degraded tier", ultraDegradedOverflowFlagKey) + } +} + +// TestRecordAfterStopIsNoop guards Finding #4 (record() can enqueue after shutdown). After +// stop(), record() must NOT enqueue into the never-drained events channel; the event must be +// counted as dropped instead. On the pre-fix code (no stopped check in record()) the event is +// enqueued and silently lost. +func TestRecordAfterStopIsNoop(t *testing.T) { + w := newFlagEvaluationWriter(ProviderConfig{}) + + // Do NOT start the worker. stop() must still be safe (ticker==nil path) and mark stopped. + w.stop() + + before := w.dropped.Load() + + // Build a minimal hook context + details to drive record(). + hookCtx := of.NewHookContext( + "test-flag", + of.Boolean, + false, + of.NewClientMetadata(""), + of.Metadata{Name: "test-provider"}, + of.NewEvaluationContext("user-1", nil), + ) + details := of.InterfaceEvaluationDetails{ + Value: true, + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "test-flag", + FlagType: of.Boolean, + ResolutionDetail: of.ResolutionDetail{ + Variant: "on", + Reason: of.TargetingMatchReason, + }, + }, + } + + w.record(hookCtx, details) + + if got := len(w.events); got != 0 { + t.Errorf("record() after stop() enqueued %d event(s); expected 0 (no-op into never-drained channel)", got) + } + if got := w.dropped.Load(); got != before+1 { + t.Errorf("record() after stop() must count the event as dropped: dropped=%d, expected=%d", got, before+1) + } +} + +// TestExtractEvalDetailsPrefersErrorMessage guards Finding #5 (error payload uses ErrorCode +// instead of OpenFeature's ErrorMessage). ErrorMessage is preferred when present; ErrorCode is +// the fallback only when ErrorMessage is empty. +func TestExtractEvalDetailsPrefersErrorMessage(t *testing.T) { + mkHookCtx := func() of.HookContext { + return of.NewHookContext( + "test-flag", + of.Boolean, + false, + of.NewClientMetadata(""), + of.Metadata{Name: "test-provider"}, + of.NewEvaluationContext("user-1", nil), + ) + } + + t.Run("ErrorMessage present is preferred over ErrorCode", func(t *testing.T) { + hc := mkHookCtx() + details := of.InterfaceEvaluationDetails{ + EvaluationDetails: of.EvaluationDetails{ + ResolutionDetail: of.ResolutionDetail{ + Reason: of.ErrorReason, + ErrorCode: of.GeneralCode, + ErrorMessage: "boom", + }, + }, + } + if got := extractEvalDetails(hc, details).errorMessage; got != "boom" { + t.Errorf("expected errorMessage=\"boom\", got %q", got) + } + }) + + t.Run("empty ErrorMessage falls back to ErrorCode", func(t *testing.T) { + hc := mkHookCtx() + details := of.InterfaceEvaluationDetails{ + EvaluationDetails: of.EvaluationDetails{ + ResolutionDetail: of.ResolutionDetail{ + Reason: of.ErrorReason, + ErrorCode: of.TypeMismatchCode, + }, + }, + } + if got := extractEvalDetails(hc, details).errorMessage; got != string(of.TypeMismatchCode) { + t.Errorf("expected errorMessage=%q (ErrorCode fallback), got %q", string(of.TypeMismatchCode), got) + } + }) + + t.Run("both empty yields empty errorMessage", func(t *testing.T) { + hc := mkHookCtx() + details := of.InterfaceEvaluationDetails{ + EvaluationDetails: of.EvaluationDetails{ + ResolutionDetail: of.ResolutionDetail{ + Reason: of.TargetingMatchReason, + }, + }, + } + if got := extractEvalDetails(hc, details).errorMessage; got != "" { + t.Errorf("expected empty errorMessage, got %q", got) + } + }) +} From 439029af9dea3eca025a8c23c1a4d9955360a2a1 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 09:00:36 -0400 Subject: [PATCH 14/37] fix(openfeature): derive runtime-default from absent variant only (PR review) - isRuntimeDefault now keys solely on an empty Variant; our evaluator sets a variant only on a matched allocation (TARGETING_MATCH/SPLIT/STATIC), so every DEFAULT/DISABLED/ERROR path already leaves it empty. - Removes the dead secondary reason-clause (DEFAULT/DISABLED) flagged as belt-and-suspenders; a present variant now correctly means a real assignment, including for foreign providers. - Aligns with PR #4886 reviewer concern: detect runtime default from absence of variant, not reason alone. --- openfeature/flagevaluation_hook.go | 10 +++----- openfeature/flagevaluation_hook_test.go | 33 ++++++++++++++++--------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/openfeature/flagevaluation_hook.go b/openfeature/flagevaluation_hook.go index b24cc2af6d6..f43838bd7d1 100644 --- a/openfeature/flagevaluation_hook.go +++ b/openfeature/flagevaluation_hook.go @@ -45,11 +45,9 @@ func (h *flagEvaluationHook) Finally( } // isRuntimeDefault returns true when the caller's supplied default value was returned. -// Primary signal: absent variant key (flag-not-found, provider-not-ready, type-mismatch, no allocation). -// Secondary: explicit DEFAULT or DISABLED reason (belt-and-suspenders). +// Signal: absent variant key. Our evaluator sets a variant ONLY on a matched allocation +// (reason TARGETING_MATCH/SPLIT/STATIC); every DEFAULT/DISABLED/ERROR path leaves the variant +// empty (see evaluator.go). A present variant therefore means a real assignment, not a default. func isRuntimeDefault(details of.InterfaceEvaluationDetails) bool { - if details.Variant == "" { - return true - } - return details.Reason == of.DefaultReason || details.Reason == of.DisabledReason + return details.Variant == "" } diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index 00a5e3d48c2..8402592edc8 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -66,10 +66,10 @@ func makeEvalDetails(variant string, reason of.Reason, errorCode of.ErrorCode, m } // TestIsRuntimeDefault verifies the runtime-default detection rule. -// Primary signal: variant=="" → true. -// Secondary signal: reason==DEFAULT or reason==DISABLED → true even if variant is non-empty. -// -// It must fail RED: isRuntimeDefault panics with "not implemented". +// Signal: absent variant key. Our evaluator sets a variant ONLY on a matched +// allocation (TARGETING_MATCH/SPLIT/STATIC); every DEFAULT/DISABLED/ERROR path +// leaves the variant empty. A present variant therefore means a real assignment, +// never a default — regardless of the reported reason. func TestIsRuntimeDefault(t *testing.T) { tests := []struct { name string @@ -82,15 +82,20 @@ func TestIsRuntimeDefault(t *testing.T) { want: true, }, { - name: "reason DEFAULT is runtime default", + name: "empty variant with DEFAULT reason is runtime default", details: makeEvalDetails("", of.DefaultReason, ""), want: true, }, { - name: "reason DISABLED is runtime default", + name: "empty variant with DISABLED reason is runtime default", details: makeEvalDetails("", of.DisabledReason, ""), want: true, }, + { + name: "empty variant with ERROR reason is runtime default", + details: makeEvalDetails("", of.ErrorReason, of.FlagNotFoundCode), + want: true, + }, { name: "variant present with TARGETING_MATCH is NOT runtime default", details: makeEvalDetails("on", of.TargetingMatchReason, ""), @@ -107,14 +112,18 @@ func TestIsRuntimeDefault(t *testing.T) { want: false, }, { - name: "empty variant with ERROR reason is runtime default (primary: variant absent)", - details: makeEvalDetails("", of.ErrorReason, of.FlagNotFoundCode), - want: true, + // Divergent case: a present variant means a real assignment even if the + // reason is DISABLED. The old secondary reason-clause wrongly returned true. + name: "variant present with DISABLED reason is NOT runtime default", + details: makeEvalDetails("on", of.DisabledReason, ""), + want: false, }, { - name: "variant present with DEFAULT reason is runtime default (secondary belt-and-suspenders)", - details: makeEvalDetails("default-variant", of.DefaultReason, ""), - want: true, + // Divergent case: a present variant means a real assignment even if the + // reason is DEFAULT. The old secondary reason-clause wrongly returned true. + name: "variant present with DEFAULT reason is NOT runtime default", + details: makeEvalDetails("on", of.DefaultReason, ""), + want: false, }, } From 20f8b9c6143fc224de21d9c25893788b4ab815ba Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 09:35:29 -0400 Subject: [PATCH 15/37] test(openfeature): table-ify EVP flagevaluation tests and benches --- openfeature/flagevaluation_hook_test.go | 153 +++--- openfeature/flagevaluation_provider_test.go | 143 +++--- openfeature/flagevaluation_test.go | 501 ++++++++++---------- openfeature/provider_bench_test.go | 179 +++---- 4 files changed, 451 insertions(+), 525 deletions(-) diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index 8402592edc8..8e3c42c3ff1 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -142,102 +142,81 @@ func TestIsRuntimeDefault(t *testing.T) { // // It must fail RED: the hook's Finally method panics with "not implemented". func TestFlagEvaluationHookFinally(t *testing.T) { - t.Run("success path records entry", func(t *testing.T) { - w := setupTestWriter(t) - hook := newFlagEvaluationHook(w) + runtimeDefaultTrue := true - hookCtx := makeHookContext("test-flag", "user-123", map[string]any{"country": "US"}) - details := makeEvalDetails("on", of.TargetingMatchReason, "", of.FlagMetadata{ - metadataAllocationKey: "default-alloc", - }) - - hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) - - // record() enqueues asynchronously; drain the one event into the aggregator so the - // assertions below observe it deterministically (no worker runs in this test). - w.aggregate(<-w.events) - - w.aggregator.mu.Lock() - defer w.aggregator.mu.Unlock() - - total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) - if total == 0 { - t.Error("expected Finally to record an entry on success path, got none") - } - }) - - t.Run("error-reason path records entry (flag-not-found)", func(t *testing.T) { - w := setupTestWriter(t) - hook := newFlagEvaluationHook(w) - - hookCtx := makeHookContext("missing-flag", "user-123", nil) - details := makeEvalDetails("", of.ErrorReason, of.FlagNotFoundCode) - - hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) - - // record() enqueues asynchronously; drain the one event into the aggregator so the - // assertions below observe it deterministically (no worker runs in this test). - w.aggregate(<-w.events) - - w.aggregator.mu.Lock() - defer w.aggregator.mu.Unlock() - - total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) - if total == 0 { - t.Error("expected Finally to record an entry on error-reason path, got none") - } - }) - - t.Run("provider-not-ready path records entry", func(t *testing.T) { - w := setupTestWriter(t) - hook := newFlagEvaluationHook(w) - - hookCtx := makeHookContext("any-flag", "", nil) - details := makeEvalDetails("", of.ErrorReason, of.ProviderNotReadyCode) - - hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) - - // record() enqueues asynchronously; drain the one event into the aggregator so the - // assertions below observe it deterministically (no worker runs in this test). - w.aggregate(<-w.events) - - w.aggregator.mu.Lock() - defer w.aggregator.mu.Unlock() - - total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) - if total == 0 { - t.Error("expected Finally to record an entry on provider-not-ready path, got none") - } - }) + tests := []struct { + name string + flagKey string + targetingKey string + attrs map[string]any + variant string + reason of.Reason + errorCode of.ErrorCode + metadata []of.FlagMetadata + wantRuntimeDefault *bool // when set, assert the recorded full-tier entry's runtimeDefault + }{ + { + name: "success path records entry", + flagKey: "test-flag", + targetingKey: "user-123", + attrs: map[string]any{"country": "US"}, + variant: "on", + reason: of.TargetingMatchReason, + metadata: []of.FlagMetadata{{metadataAllocationKey: "default-alloc"}}, + }, + { + name: "error-reason path records entry (flag-not-found)", + flagKey: "missing-flag", + targetingKey: "user-123", + reason: of.ErrorReason, + errorCode: of.FlagNotFoundCode, + }, + { + name: "provider-not-ready path records entry", + flagKey: "any-flag", + reason: of.ErrorReason, + errorCode: of.ProviderNotReadyCode, + }, + { + name: "DEFAULT reason path records entry with runtime_default_used=true", + flagKey: "absent-flag", + targetingKey: "user-456", + reason: of.DefaultReason, + wantRuntimeDefault: &runtimeDefaultTrue, + }, + } - t.Run("DEFAULT reason path records entry with runtime_default_used=true", func(t *testing.T) { - w := setupTestWriter(t) - hook := newFlagEvaluationHook(w) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + w := setupTestWriter(t) + hook := newFlagEvaluationHook(w) - hookCtx := makeHookContext("absent-flag", "user-456", nil) - details := makeEvalDetails("", of.DefaultReason, "") + hookCtx := makeHookContext(tc.flagKey, tc.targetingKey, tc.attrs) + details := makeEvalDetails(tc.variant, tc.reason, tc.errorCode, tc.metadata...) - hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) + hook.Finally(context.Background(), hookCtx, details, of.HookHints{}) - // record() enqueues asynchronously; drain the one event into the aggregator so the - // assertions below observe it deterministically (no worker runs in this test). - w.aggregate(<-w.events) + // record() enqueues asynchronously; drain the one event into the aggregator so the + // assertions below observe it deterministically (no worker runs in this test). + w.aggregate(<-w.events) - w.aggregator.mu.Lock() - defer w.aggregator.mu.Unlock() + w.aggregator.mu.Lock() + defer w.aggregator.mu.Unlock() - total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) - if total == 0 { - t.Error("expected Finally to record an entry on DEFAULT-reason path, got none") - } + total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) + if total == 0 { + t.Error("expected Finally to record an entry, got none") + } - // The recorded entry should have runtimeDefault=true - for _, e := range w.aggregator.full { - if !e.runtimeDefault { - t.Error("expected runtimeDefault=true for DEFAULT-reason evaluation") + if tc.wantRuntimeDefault != nil { + for _, e := range w.aggregator.full { + if e.runtimeDefault != *tc.wantRuntimeDefault { + t.Errorf("expected runtimeDefault=%v, got %v", *tc.wantRuntimeDefault, e.runtimeDefault) + } + } } - } - }) + }) + } } // TestFlagEvaluationHookContextCancelled verifies that a cancelled context does NOT diff --git a/openfeature/flagevaluation_provider_test.go b/openfeature/flagevaluation_provider_test.go index 541fd4cf243..3b81fd12cab 100644 --- a/openfeature/flagevaluation_provider_test.go +++ b/openfeature/flagevaluation_provider_test.go @@ -6,7 +6,6 @@ package openfeature import ( - "reflect" "testing" of "github.com/open-feature/go-sdk/openfeature" @@ -20,90 +19,78 @@ import ( // When the killswitch is unset or "true": the EVP hook IS registered. // The OTel flagEvalHook is present in Hooks() in BOTH cases. func TestFlagEvaluationKillswitch(t *testing.T) { - t.Run("killswitch disabled: EVP hook absent from Hooks(), OTel hook present", func(t *testing.T) { - t.Setenv(flagEvalCountsEnabledEnvVar, "false") - - p := newDatadogProvider(ProviderConfig{}) - - if p.flagEvalWriter != nil { - t.Error("expected flagEvalWriter to be nil when killswitch is disabled") - } - if p.flagEvalEVPHook != nil { - t.Error("expected flagEvalEVPHook to be nil when killswitch is disabled") - } - - hooks := p.Hooks() - - // OTel hook must still be present. - otelPresent := false - for _, h := range hooks { - if _, ok := h.(*flagEvalHook); ok { - otelPresent = true + tests := []struct { + name string + envValue string + wantEVPEnabled bool + }{ + { + name: "killswitch disabled: EVP hook absent from Hooks(), OTel hook present", + envValue: "false", + wantEVPEnabled: false, + }, + { + // "1" exercises the default-true behavior (any truthy value enables the EVP path). + name: "killswitch enabled (unset = default true): EVP hook present in Hooks()", + envValue: "1", + wantEVPEnabled: true, + }, + { + name: "killswitch explicitly true: EVP hook present in Hooks()", + envValue: "true", + wantEVPEnabled: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(flagEvalCountsEnabledEnvVar, tc.envValue) + + p := newDatadogProvider(ProviderConfig{}) + + if tc.wantEVPEnabled { + if p.flagEvalWriter == nil { + t.Error("expected flagEvalWriter to be non-nil when killswitch is enabled") + } + if p.flagEvalEVPHook == nil { + t.Error("expected flagEvalEVPHook to be non-nil when killswitch is enabled") + } + } else { + if p.flagEvalWriter != nil { + t.Error("expected flagEvalWriter to be nil when killswitch is disabled") + } + if p.flagEvalEVPHook != nil { + t.Error("expected flagEvalEVPHook to be nil when killswitch is disabled") + } } - } - if !otelPresent { - t.Error("expected OTel flagEvalHook to be present in Hooks() even when killswitch is disabled") - } - // EVP hook must be absent. - for _, h := range hooks { - if _, ok := h.(*flagEvaluationHook); ok { - t.Errorf("expected EVP flagEvaluationHook to be absent from Hooks() when killswitch is disabled, but found one: %v", reflect.TypeOf(h)) + hooks := p.Hooks() + + otelPresent := false + evpPresent := false + for _, h := range hooks { + switch h.(type) { + case *flagEvalHook: + otelPresent = true + case *flagEvaluationHook: + evpPresent = true + } } - } - }) - - t.Run("killswitch enabled (unset = default true): EVP hook present in Hooks()", func(t *testing.T) { - // Ensure the env var is unset to test the default-true behavior. - t.Setenv(flagEvalCountsEnabledEnvVar, "1") - - p := newDatadogProvider(ProviderConfig{}) - if p.flagEvalWriter == nil { - t.Error("expected flagEvalWriter to be non-nil when killswitch is enabled (default)") - } - if p.flagEvalEVPHook == nil { - t.Error("expected flagEvalEVPHook to be non-nil when killswitch is enabled (default)") - } - - hooks := p.Hooks() - - // Both OTel and EVP hooks must be present. - otelPresent := false - evpPresent := false - for _, h := range hooks { - switch h.(type) { - case *flagEvalHook: - otelPresent = true - case *flagEvaluationHook: - evpPresent = true + // The OTel hook must be present in EVERY case — the killswitch never affects it. + if !otelPresent { + t.Error("expected OTel flagEvalHook to be present in Hooks() regardless of the killswitch") } - } - if !otelPresent { - t.Error("expected OTel flagEvalHook to be present in Hooks() when killswitch is enabled") - } - if !evpPresent { - t.Error("expected EVP flagEvaluationHook to be present in Hooks() when killswitch is enabled") - } - }) - - t.Run("killswitch explicitly true: EVP hook present in Hooks()", func(t *testing.T) { - t.Setenv(flagEvalCountsEnabledEnvVar, "true") - - p := newDatadogProvider(ProviderConfig{}) - - hooks := p.Hooks() - evpPresent := false - for _, h := range hooks { - if _, ok := h.(*flagEvaluationHook); ok { - evpPresent = true + if evpPresent != tc.wantEVPEnabled { + if tc.wantEVPEnabled { + t.Error("expected EVP flagEvaluationHook to be present in Hooks() when killswitch is enabled") + } else { + t.Errorf("expected EVP flagEvaluationHook to be absent from Hooks() when killswitch is disabled, but found one") + } } - } - if !evpPresent { - t.Error("expected EVP flagEvaluationHook to be present in Hooks() when killswitch is explicitly 'true'") - } - }) + }) + } } // Compile-time assertion: flagEvaluationHook implements the OpenFeature Hook interface. diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 141543c1418..c539c91e6aa 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -32,68 +32,103 @@ func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { } } +// newTestAggregator builds a flagEvaluationAggregator with explicit, caller-supplied caps. +// Unlike setupTestAggregator (which fixes small caps), each cap is a parameter so a test can +// drive a specific tier-overflow scenario. The cap NUMBERS are load-bearing — callers pass +// the exact values their scenario requires. +func newTestAggregator(globalCap, perFlagCap, degradedCap, ultraDegCap int) *flagEvaluationAggregator { + return &flagEvaluationAggregator{ + full: make(map[evaluationAggregationKey]*evaluationEntry), + degraded: make(map[evaluationDegradedKey]*evaluationEntry), + ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), + perFlagFull: make(map[string]int), + globalCap: globalCap, + perFlagCap: perFlagCap, + degradedCap: degradedCap, + ultraDegCap: ultraDegCap, + } +} + // TestPruneContext verifies that pruneContext applies the 256-field / 256-char limits // before evaluation context enters the aggregation buffer. // // It must fail RED: pruneContext panics with "not implemented". func TestPruneContext(t *testing.T) { - t.Run("300 fields truncated to exactly 256", func(t *testing.T) { - raw := make(map[string]any, 300) - for i := range 300 { - raw[fmt.Sprintf("key%d", i)] = fmt.Sprintf("value%d", i) - } - - out := pruneContext(raw) - - if len(out) != 256 { - t.Errorf("expected exactly 256 fields after prune, got %d", len(out)) - } - }) - - t.Run("string value exceeding 256 chars is dropped", func(t *testing.T) { - longVal := strings.Repeat("x", 300) // 300 chars > maxFieldLength(256) - raw := map[string]any{ - "short": "ok", - "long": longVal, - } - - out := pruneContext(raw) - - if _, ok := out["long"]; ok { - t.Error("expected long string value to be dropped from pruned context") - } - if _, ok := out["short"]; !ok { - t.Error("expected short string value to be retained in pruned context") - } - }) - - t.Run("nil input returns nil", func(t *testing.T) { - out := pruneContext(nil) - if out != nil { - t.Errorf("expected nil for nil input, got %v", out) - } - }) - - t.Run("empty input returns nil or empty", func(t *testing.T) { - out := pruneContext(map[string]any{}) - if out != nil && len(out) != 0 { - t.Errorf("expected nil or empty for empty input, got %v", out) - } - }) + tests := []struct { + name string + input map[string]any + assert func(t *testing.T, out map[string]any) + }{ + { + name: "300 fields truncated to exactly 256", + input: func() map[string]any { + raw := make(map[string]any, 300) + for i := range 300 { + raw[fmt.Sprintf("key%d", i)] = fmt.Sprintf("value%d", i) + } + return raw + }(), + assert: func(t *testing.T, out map[string]any) { + if len(out) != 256 { + t.Errorf("expected exactly 256 fields after prune, got %d", len(out)) + } + }, + }, + { + name: "string value exceeding 256 chars is dropped", + input: map[string]any{ + "short": "ok", + "long": strings.Repeat("x", 300), // 300 chars > maxFieldLength(256) + }, + assert: func(t *testing.T, out map[string]any) { + if _, ok := out["long"]; ok { + t.Error("expected long string value to be dropped from pruned context") + } + if _, ok := out["short"]; !ok { + t.Error("expected short string value to be retained in pruned context") + } + }, + }, + { + name: "nil input returns nil", + input: nil, + assert: func(t *testing.T, out map[string]any) { + if out != nil { + t.Errorf("expected nil for nil input, got %v", out) + } + }, + }, + { + name: "empty input returns nil or empty", + input: map[string]any{}, + assert: func(t *testing.T, out map[string]any) { + if out != nil && len(out) != 0 { + t.Errorf("expected nil or empty for empty input, got %v", out) + } + }, + }, + { + name: "non-string values are retained regardless of notional length", + input: map[string]any{ + "intVal": 42, + "boolVal": true, + }, + assert: func(t *testing.T, out map[string]any) { + if out["intVal"] == nil { + t.Error("expected integer value to be retained") + } + if out["boolVal"] == nil { + t.Error("expected boolean value to be retained") + } + }, + }, + } - t.Run("non-string values are retained regardless of notional length", func(t *testing.T) { - raw := map[string]any{ - "intVal": 42, - "boolVal": true, - } - out := pruneContext(raw) - if out["intVal"] == nil { - t.Error("expected integer value to be retained") - } - if out["boolVal"] == nil { - t.Error("expected boolean value to be retained") - } - }) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.assert(t, pruneContext(tc.input)) + }) + } } // TestFlagEvaluationPayloadSchema verifies that full, degraded, and ultra-degraded events @@ -104,122 +139,90 @@ func TestPruneContext(t *testing.T) { func TestFlagEvaluationPayloadSchema(t *testing.T) { nowMs := time.Now().UnixMilli() - t.Run("full tier has all required fields", func(t *testing.T) { - event := flagEvaluationEvent{ - Timestamp: nowMs, - Flag: flagEvalFlag{Key: "test-flag"}, - FirstEvaluation: nowMs, - LastEvaluation: nowMs, - EvaluationCount: 1, - Variant: &flagEvalVariant{Key: "on"}, - TargetingKey: "user-123", - Context: &flagEvalEventContext{ - Evaluation: map[string]any{"country": "US"}, + requiredFields := []string{"timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count"} + + tierTests := []struct { + name string + event flagEvaluationEvent + requiredFlgKey bool // full tier additionally asserts flag.key is present + optionalAbsent []string // optional fields that must NOT appear for this tier + }{ + { + name: "full tier has all required fields", + event: flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "test-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 1, + Variant: &flagEvalVariant{Key: "on"}, + TargetingKey: "user-123", + Context: &flagEvalEventContext{ + Evaluation: map[string]any{"country": "US"}, + }, }, - } - - b, err := json.Marshal(event) - if err != nil { - t.Fatalf("failed to marshal full event: %v", err) - } - - var m map[string]any - if err := json.Unmarshal(b, &m); err != nil { - t.Fatalf("failed to unmarshal: %v", err) - } + requiredFlgKey: true, + }, + { + name: "degraded tier omits targeting_key and context.evaluation", + // Degraded tier: no targeting_key, no context.evaluation; variant + allocation present. + event: flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "test-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 5, + Variant: &flagEvalVariant{Key: "on"}, + Allocation: &flagEvalAllocation{Key: "default"}, + // TargetingKey / Context intentionally absent. + }, + optionalAbsent: []string{"targeting_key", "context"}, + }, + { + name: "ultra-degraded tier has only required fields", + // Ultra-degraded: only flag key + counts; no variant, allocation, targeting, context. + event: flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "test-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 1000, + // All optional fields intentionally absent. + }, + optionalAbsent: []string{"targeting_key", "variant", "allocation", "targeting_rule", "error", "context", "runtime_default_used"}, + }, + } - // Required fields must be present - for _, req := range []string{"timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count"} { - if _, ok := m[req]; !ok { - t.Errorf("full tier: required field %q missing from marshaled JSON", req) + for _, tc := range tierTests { + t.Run(tc.name, func(t *testing.T) { + b, err := json.Marshal(tc.event) + if err != nil { + t.Fatalf("failed to marshal event: %v", err) } - } - - // flag.key must be present - if flagObj, ok := m["flag"].(map[string]any); !ok { - t.Error("full tier: flag is not an object") - } else if _, ok := flagObj["key"]; !ok { - t.Error("full tier: flag.key missing") - } - }) - - t.Run("degraded tier omits targeting_key and context.evaluation", func(t *testing.T) { - // Degraded tier: no targeting_key, no context.evaluation; variant + allocation present - event := flagEvaluationEvent{ - Timestamp: nowMs, - Flag: flagEvalFlag{Key: "test-flag"}, - FirstEvaluation: nowMs, - LastEvaluation: nowMs, - EvaluationCount: 5, - Variant: &flagEvalVariant{Key: "on"}, - Allocation: &flagEvalAllocation{Key: "default"}, - // TargetingKey: intentionally absent - // Context: intentionally absent - } - - b, err := json.Marshal(event) - if err != nil { - t.Fatalf("failed to marshal degraded event: %v", err) - } - - var m map[string]any - if err := json.Unmarshal(b, &m); err != nil { - t.Fatalf("failed to unmarshal: %v", err) - } - - // Required fields must be present - for _, req := range []string{"timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count"} { - if _, ok := m[req]; !ok { - t.Errorf("degraded tier: required field %q missing", req) + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("failed to unmarshal: %v", err) } - } - - // targeting_key must be absent - if _, ok := m["targeting_key"]; ok { - t.Error("degraded tier: targeting_key should be absent") - } - - // context must be absent - if _, ok := m["context"]; ok { - t.Error("degraded tier: context should be absent") - } - }) - - t.Run("ultra-degraded tier has only required fields", func(t *testing.T) { - // Ultra-degraded: only flag key + counts; no variant, allocation, targeting, context - event := flagEvaluationEvent{ - Timestamp: nowMs, - Flag: flagEvalFlag{Key: "test-flag"}, - FirstEvaluation: nowMs, - LastEvaluation: nowMs, - EvaluationCount: 1000, - // All optional fields intentionally absent - } - - b, err := json.Marshal(event) - if err != nil { - t.Fatalf("failed to marshal ultra-degraded event: %v", err) - } - var m map[string]any - if err := json.Unmarshal(b, &m); err != nil { - t.Fatalf("failed to unmarshal: %v", err) - } - - // Required fields must be present - for _, req := range []string{"timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count"} { - if _, ok := m[req]; !ok { - t.Errorf("ultra-degraded tier: required field %q missing", req) + for _, req := range requiredFields { + if _, ok := m[req]; !ok { + t.Errorf("required field %q missing from marshaled JSON", req) + } } - } - - // Optional fields must be absent - for _, opt := range []string{"targeting_key", "variant", "allocation", "targeting_rule", "error", "context", "runtime_default_used"} { - if _, ok := m[opt]; ok { - t.Errorf("ultra-degraded tier: optional field %q should be absent", opt) + if tc.requiredFlgKey { + if flagObj, ok := m["flag"].(map[string]any); !ok { + t.Error("flag is not an object") + } else if _, ok := flagObj["key"]; !ok { + t.Error("flag.key missing") + } } - } - }) + for _, opt := range tc.optionalAbsent { + if _, ok := m[opt]; ok { + t.Errorf("optional field %q should be absent", opt) + } + } + }) + } t.Run("first_evaluation and last_evaluation meet minimum constraint", func(t *testing.T) { // Schema minimum: 1759276800000 (2025-08-01 Unix ms) @@ -318,16 +321,8 @@ func TestAggregatorCollisionSafety(t *testing.T) { // produce count==1000 and firstEvaluation<=lastEvaluation. // Must be run with -race to satisfy the race-free requirement. func TestAggregatorConcurrentMinMax(t *testing.T) { - agg := &flagEvaluationAggregator{ - full: make(map[evaluationAggregationKey]*evaluationEntry), - degraded: make(map[evaluationDegradedKey]*evaluationEntry), - ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), - perFlagFull: make(map[string]int), - globalCap: 100_000, // large enough not to overflow during this test - perFlagCap: 100_000, - degradedCap: 100_000, - ultraDegCap: defaultEvalUltraDegradedCap, - } + // Caps large enough not to overflow during this test. + agg := newTestAggregator(100_000, 100_000, 100_000, defaultEvalUltraDegradedCap) d := evalDetails{ flagKey: "concurrent-flag", @@ -378,16 +373,7 @@ func TestSaturationCountPreservation(t *testing.T) { // globalCap=5 means only 5 full-tier buckets ever created. // perFlagCap=2 means after 2 distinct full-tier buckets per flag, it overflows to degraded. // degradedCap=3 means only 3 degraded buckets; further overflow goes to ultra-degraded. - agg := &flagEvaluationAggregator{ - full: make(map[evaluationAggregationKey]*evaluationEntry), - degraded: make(map[evaluationDegradedKey]*evaluationEntry), - ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), - perFlagFull: make(map[string]int), - globalCap: 5, - perFlagCap: 2, - degradedCap: 3, - ultraDegCap: defaultEvalUltraDegradedCap, - } + agg := newTestAggregator(5, 2, 3, defaultEvalUltraDegradedCap) nowMs := time.Now().UnixMilli() // We drive 100 distinct evaluations. Each call to add() must contribute exactly 1 @@ -643,26 +629,39 @@ func TestPruneContextOversizedStringDeterministic(t *testing.T) { // values/keys must not be able to fake a multi-field context. Drives a subset through // aggregator.add and asserts SEPARATE full-tier buckets. func TestHashContextCanonicalEncoding(t *testing.T) { - t.Run("int 1 != string 1", func(t *testing.T) { - if hashContext(map[string]any{"x": 1}) == hashContext(map[string]any{"x": "1"}) { - t.Error("type-tagged encoding must distinguish int 1 from string \"1\"") - } - }) - - t.Run("'=' in key/value cannot alias a multi-field context", func(t *testing.T) { - // {"a=b":"c"} vs {"a":"b=c"} render identically under key+"="+value with no - // length delimiter; canonical encoding must keep them distinct. - if hashContext(map[string]any{"a=b": "c"}) == hashContext(map[string]any{"a": "b=c"}) { - t.Error("'=' in key/value must not alias across fields") - } - }) + // Distinct contexts must hash differently — no aliasing across type or delimiter tricks. + inequalityTests := []struct { + name string + mapA, mapB map[string]any + }{ + { + // Type-tagged encoding must distinguish int 1 from string "1". + name: "int 1 != string 1", + mapA: map[string]any{"x": 1}, + mapB: map[string]any{"x": "1"}, + }, + { + // {"a=b":"c"} vs {"a":"b=c"} render identically under key+"="+value with no + // length delimiter; canonical encoding must keep them distinct. + name: "'=' in key/value cannot alias a multi-field context", + mapA: map[string]any{"a=b": "c"}, + mapB: map[string]any{"a": "b=c"}, + }, + { + // Under key+"="+value+"\n", a newline in a value would collide with a two-field map. + name: "'\\n' in value cannot alias a multi-field context", + mapA: map[string]any{"a": "x\ny", "b": "z"}, + mapB: map[string]any{"a": "x", "y=z": ""}, + }, + } - t.Run("'\\n' in value cannot alias a multi-field context", func(t *testing.T) { - // {"a":"x\ny":"z"} would, under key+"="+value+"\n", collide with a two-field map. - if hashContext(map[string]any{"a": "x\ny", "b": "z"}) == hashContext(map[string]any{"a": "x", "y=z": ""}) { - t.Error("'\\n' in value must not alias a multi-field context") - } - }) + for _, tc := range inequalityTests { + t.Run(tc.name, func(t *testing.T) { + if hashContext(tc.mapA) == hashContext(tc.mapB) { + t.Errorf("canonical encoding must distinguish %v from %v", tc.mapA, tc.mapB) + } + }) + } t.Run("distinct contexts land in separate full-tier buckets via add()", func(t *testing.T) { agg := setupTestAggregator(t) @@ -685,16 +684,8 @@ func TestHashContextCanonicalEncoding(t *testing.T) { // preserved (Σ across tiers == add() calls). func TestUltraDegradedCapBounded(t *testing.T) { const cap = 3 - agg := &flagEvaluationAggregator{ - full: make(map[evaluationAggregationKey]*evaluationEntry), - degraded: make(map[evaluationDegradedKey]*evaluationEntry), - ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), - perFlagFull: make(map[string]int), - globalCap: 0, // force everything straight into the ultra-degraded tier - perFlagCap: 100_000, - degradedCap: 0, - ultraDegCap: cap, - } + // globalCap=0 and degradedCap=0 force everything straight into the ultra-degraded tier. + agg := newTestAggregator(0, 100_000, 0, cap) nowMs := time.Now().UnixMilli() const calls = 100 @@ -792,48 +783,54 @@ func TestExtractEvalDetailsPrefersErrorMessage(t *testing.T) { ) } - t.Run("ErrorMessage present is preferred over ErrorCode", func(t *testing.T) { - hc := mkHookCtx() - details := of.InterfaceEvaluationDetails{ - EvaluationDetails: of.EvaluationDetails{ - ResolutionDetail: of.ResolutionDetail{ - Reason: of.ErrorReason, - ErrorCode: of.GeneralCode, - ErrorMessage: "boom", + tests := []struct { + name string + details of.InterfaceEvaluationDetails + wantErrorMessage string + }{ + { + name: "ErrorMessage present is preferred over ErrorCode", + details: of.InterfaceEvaluationDetails{ + EvaluationDetails: of.EvaluationDetails{ + ResolutionDetail: of.ResolutionDetail{ + Reason: of.ErrorReason, + ErrorCode: of.GeneralCode, + ErrorMessage: "boom", + }, }, }, - } - if got := extractEvalDetails(hc, details).errorMessage; got != "boom" { - t.Errorf("expected errorMessage=\"boom\", got %q", got) - } - }) - - t.Run("empty ErrorMessage falls back to ErrorCode", func(t *testing.T) { - hc := mkHookCtx() - details := of.InterfaceEvaluationDetails{ - EvaluationDetails: of.EvaluationDetails{ - ResolutionDetail: of.ResolutionDetail{ - Reason: of.ErrorReason, - ErrorCode: of.TypeMismatchCode, + wantErrorMessage: "boom", + }, + { + name: "empty ErrorMessage falls back to ErrorCode", + details: of.InterfaceEvaluationDetails{ + EvaluationDetails: of.EvaluationDetails{ + ResolutionDetail: of.ResolutionDetail{ + Reason: of.ErrorReason, + ErrorCode: of.TypeMismatchCode, + }, }, }, - } - if got := extractEvalDetails(hc, details).errorMessage; got != string(of.TypeMismatchCode) { - t.Errorf("expected errorMessage=%q (ErrorCode fallback), got %q", string(of.TypeMismatchCode), got) - } - }) - - t.Run("both empty yields empty errorMessage", func(t *testing.T) { - hc := mkHookCtx() - details := of.InterfaceEvaluationDetails{ - EvaluationDetails: of.EvaluationDetails{ - ResolutionDetail: of.ResolutionDetail{ - Reason: of.TargetingMatchReason, + wantErrorMessage: string(of.TypeMismatchCode), + }, + { + name: "both empty yields empty errorMessage", + details: of.InterfaceEvaluationDetails{ + EvaluationDetails: of.EvaluationDetails{ + ResolutionDetail: of.ResolutionDetail{ + Reason: of.TargetingMatchReason, + }, }, }, - } - if got := extractEvalDetails(hc, details).errorMessage; got != "" { - t.Errorf("expected empty errorMessage, got %q", got) - } - }) + wantErrorMessage: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := extractEvalDetails(mkHookCtx(), tc.details).errorMessage; got != tc.wantErrorMessage { + t.Errorf("errorMessage = %q, want %q", got, tc.wantErrorMessage) + } + }) + } } diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 6a8a825e865..7ba4f0b13ac 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -15,103 +15,69 @@ import ( "github.com/open-feature/go-sdk/openfeature" ) -// BenchmarkBooleanEvaluation benchmarks boolean flag evaluation -func BenchmarkBooleanEvaluation(b *testing.B) { - provider := newDatadogProvider(ProviderConfig{}) - config := createTestConfig() - provider.updateConfiguration(config) - - ctx := context.Background() - flatCtx := openfeature.FlattenedContext{ - "targetingKey": "user-123", - "country": "US", - } - - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - _ = provider.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) - } -} - -// BenchmarkStringEvaluation benchmarks string flag evaluation -func BenchmarkStringEvaluation(b *testing.B) { - provider := newDatadogProvider(ProviderConfig{}) - config := createTestConfig() - provider.updateConfiguration(config) - - ctx := context.Background() - flatCtx := openfeature.FlattenedContext{ - "targetingKey": "user-123", - "age": 25, - } - - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - _ = provider.StringEvaluation(ctx, "string-flag", "default", flatCtx) - } -} - -// BenchmarkIntEvaluation benchmarks integer flag evaluation -func BenchmarkIntEvaluation(b *testing.B) { - provider := newDatadogProvider(ProviderConfig{}) - config := createTestConfig() - provider.updateConfiguration(config) - - ctx := context.Background() - flatCtx := openfeature.FlattenedContext{ - "targetingKey": "user-123", - "premium": "yes", - } - - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - _ = provider.IntEvaluation(ctx, "int-flag", 5, flatCtx) - } -} - -// BenchmarkFloatEvaluation benchmarks float flag evaluation -func BenchmarkFloatEvaluation(b *testing.B) { - provider := newDatadogProvider(ProviderConfig{}) - config := createTestConfig() - provider.updateConfiguration(config) - - ctx := context.Background() - flatCtx := openfeature.FlattenedContext{ - "targetingKey": "user-123", - "tier": "premium", - } - - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - _ = provider.FloatEvaluation(ctx, "float-flag", 0.0, flatCtx) +// BenchmarkEvaluationByType benchmarks each typed evaluation method. Every sub-benchmark +// shares the same provider/config setup and differs only in the eval method, default value, +// flag key, and one distinct flatCtx field — exactly as the former per-type BenchmarkXEvaluation +// functions did. The parent name is deliberately OUTSIDE the BenchmarkFlagEvaluation prefix so +// the EVP overhead filter (-bench='^BenchmarkFlagEvaluation') never selects it. +func BenchmarkEvaluationByType(b *testing.B) { + cases := []struct { + name string + flatCtx openfeature.FlattenedContext + eval func(p *DatadogProvider, ctx context.Context, flatCtx openfeature.FlattenedContext) + }{ + { + name: "Boolean", + flatCtx: openfeature.FlattenedContext{"targetingKey": "user-123", "country": "US"}, + eval: func(p *DatadogProvider, ctx context.Context, flatCtx openfeature.FlattenedContext) { + _ = p.BooleanEvaluation(ctx, "bool-flag", false, flatCtx) + }, + }, + { + name: "String", + flatCtx: openfeature.FlattenedContext{"targetingKey": "user-123", "age": 25}, + eval: func(p *DatadogProvider, ctx context.Context, flatCtx openfeature.FlattenedContext) { + _ = p.StringEvaluation(ctx, "string-flag", "default", flatCtx) + }, + }, + { + name: "Int", + flatCtx: openfeature.FlattenedContext{"targetingKey": "user-123", "premium": "yes"}, + eval: func(p *DatadogProvider, ctx context.Context, flatCtx openfeature.FlattenedContext) { + _ = p.IntEvaluation(ctx, "int-flag", 5, flatCtx) + }, + }, + { + name: "Float", + flatCtx: openfeature.FlattenedContext{"targetingKey": "user-123", "tier": "premium"}, + eval: func(p *DatadogProvider, ctx context.Context, flatCtx openfeature.FlattenedContext) { + _ = p.FloatEvaluation(ctx, "float-flag", 0.0, flatCtx) + }, + }, + { + name: "Object", + flatCtx: openfeature.FlattenedContext{"targetingKey": "user-123", "requests": 1500}, + eval: func(p *DatadogProvider, ctx context.Context, flatCtx openfeature.FlattenedContext) { + _ = p.ObjectEvaluation(ctx, "json-flag", nil, flatCtx) + }, + }, } -} -// BenchmarkObjectEvaluation benchmarks object flag evaluation -func BenchmarkEvaluation(b *testing.B) { - provider := newDatadogProvider(ProviderConfig{}) - config := createTestConfig() - provider.updateConfiguration(config) + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + provider := newDatadogProvider(ProviderConfig{}) + provider.updateConfiguration(createTestConfig()) - ctx := context.Background() - flatCtx := openfeature.FlattenedContext{ - "targetingKey": "user-123", - "requests": 1500, - } + ctx := context.Background() + flatCtx := tc.flatCtx - b.ReportAllocs() - b.ResetTimer() + b.ReportAllocs() + b.ResetTimer() - for b.Loop() { - _ = provider.ObjectEvaluation(ctx, "json-flag", nil, flatCtx) + for b.Loop() { + tc.eval(provider, ctx, flatCtx) + } + }) } } @@ -312,16 +278,15 @@ var flagEvalBenchProfiles = []flagEvalBenchProfile{ {"stress/10flags_1000users_250fields", 10, 1000, 250}, } -// runFlagEvalBenchmark drives evaluations through a real OpenFeature client so the -// provider's registered hooks actually run. Calling provider.BooleanEvaluation directly -// would bypass Hooks(), so neither the OTel nor the EVP hook would execute and all three -// columns would measure the same bare evaluator. Both the evaluated flag key and the -// targeting key rotate, so flag- and user-cardinality are exercised. +// runFlagEvalBenchmark drives evaluations through a real OpenFeature client so the provider's +// registered hooks actually run — calling provider.BooleanEvaluation directly would bypass +// Hooks() and every column would measure the same bare evaluator. Flag and targeting keys +// rotate to exercise flag- and user-cardinality. // -// configureHooks nils whichever provider hooks the tier under test should exclude. It runs -// before the provider is registered; Init does not recreate hooks, so the choice sticks. -// The provider uses a 24h flush interval, so the EVP writer never flushes during the run — -// no HTTP round-trip enters the hot path, isolating hook + aggregator cost. +// configureHooks nils whichever provider hooks the tier under test should exclude (it runs +// before registration; Init does not recreate hooks, so the choice sticks). The 24h flush +// interval keeps the EVP writer from flushing during the run, so no HTTP round-trip enters +// the hot path — isolating hook + aggregator cost. func runFlagEvalBenchmark(b *testing.B, configureHooks func(p *DatadogProvider)) { b.Helper() for _, p := range flagEvalBenchProfiles { @@ -393,13 +358,11 @@ func BenchmarkFlagEvaluationOTelPlusEVP(b *testing.B) { }) } -// BenchmarkFlagEvaluationEVPRecord isolates the EVP hook's synchronous hot-path cost — the -// scalar extraction + shallow context copy (EvaluationContext().Attributes()) + non-blocking -// enqueue that runs on the evaluation goroutine. A discard drainer empties the queue so it -// never fills, and the asynchronous aggregation (flatten/prune/hash/add, performed by the -// real worker off the hot path) is NOT attributed here. This is the latency a flag -// evaluation actually pays when the EVP path is enabled — distinct from the client -// benchmarks above, whose process-wide allocs/wall-clock also capture the worker's work. +// BenchmarkFlagEvaluationEVPRecord isolates the EVP hook's synchronous hot-path cost: scalar +// extraction + shallow context copy + non-blocking enqueue, all on the evaluation goroutine. +// A discard drainer keeps the queue empty so the asynchronous aggregation (flatten/prune/hash/ +// add, done by the real worker off the hot path) is NOT attributed here — this is the latency a +// flag evaluation actually pays when the EVP path is enabled. func BenchmarkFlagEvaluationEVPRecord(b *testing.B) { for _, p := range flagEvalBenchProfiles { b.Run(p.name, func(b *testing.B) { From 432002c750e1e8203560ae5293889583c168f636 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 09:35:42 -0400 Subject: [PATCH 16/37] refactor(openfeature): tighten EVP flagevaluation writer/hook in place --- openfeature/flagevaluation.go | 216 ++++++++++++++-------------------- 1 file changed, 86 insertions(+), 130 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index e798328d12f..2e8fbb500a7 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -68,44 +68,24 @@ const ( defaultEvalEventBufferSize = 4096 ) -// evaluationAggregationKey identifies one full-tier aggregation bucket. Its identity is -// split into two parts with very different collision properties: +// evaluationAggregationKey identifies one full-tier aggregation bucket. Its identity splits +// into two parts with different collision properties: // // 1. Enumerable dimensions — flagKey, variant, allocationKey, reason, targetingKey — are -// stored as EXACT string fields and compared byte-for-byte by Go map equality. Two -// structurally distinct values can NEVER alias on these dimensions. In particular, a -// count can never be attributed to the wrong flag/variant/reason/allocation. +// EXACT string fields compared byte-for-byte by Go map equality, so they can NEVER alias: +// a count is never attributed to the wrong flag/variant/reason/allocation. // -// 2. contextHash is a 64-bit FNV-1a digest of the pruned evaluation context (see -// hashContext). The context is map[string]any and therefore not comparable, so it -// cannot be a struct field — the digest stands in for it. The digest is computed over a -// CANONICAL encoding: keys are sorted (deterministic ordering), and each field is -// written as a length-delimited key plus a type-tagged, length-delimited value. Distinct -// contexts therefore cannot ALIAS by construction — int 1 vs string "1" carry different -// type tags, and '=' / '\n' inside a key or value cannot fake a field boundary. +// 2. contextHash is a 64-bit FNV-1a digest of the pruned context (map[string]any, not +// comparable, so it cannot be a struct field — see hashContext for the canonical encoding +// that makes distinct contexts non-aliasing). // -// What a contextHash collision actually does — and why it is acceptable: -// -// With the canonical encoding above, a collision is no longer a systematic encoding alias; -// it is a genuine 64-bit FNV-1a hash collision: two evaluations identical on ALL FIVE -// exact dimensions AND whose *different* canonical encodings happen to hash to the same -// uint64. When that occurs, the second evaluation matches the existing bucket on add()'s -// fast path and increments its count (the `if e, ok := a.full[fullKey]; ok` branch below). -// Consequences: -// -// - COUNT-PRESERVING: the evaluation count is merged into the existing bucket, never -// dropped. The invariant Σ(counts across all tiers) == number of add() calls still -// holds. This is exactly what TestSaturationCountPreservation guards. -// - NO MISATTRIBUTION: the count still belongs to the correct flag/variant/reason/ -// allocation — those dimensions are exact, not hashed. -// - SOLE CASUALTY is context-attribute fidelity: the bucket retains the FIRST context's -// attrs, so the colliding evaluation's distinct attrs are not separately reported. -// Context is a best-effort dimension that the degraded/ultra tiers drop entirely by -// design, so a collision is strictly less lossy than ordinary degradation. -// -// This is internal telemetry over the customer's own context — not a trust boundary — so a -// keyed/crypto hash (e.g. SipHash) is unwarranted, and a wider (128-bit) digest would buy -// only marginal context-label fidelity. Deliberately not done. +// A contextHash collision is therefore a genuine 64-bit hash collision, not an encoding alias: +// the second evaluation merges into the existing bucket via add()'s fast path. This is +// count-preserving (Σ(counts across tiers) == add() calls — TestSaturationCountPreservation) +// and never misattributing (the exact dimensions still match); the sole casualty is +// context-attribute fidelity, which the degraded/ultra tiers already drop by design. This is +// internal telemetry over the customer's own context, not a trust boundary, so a keyed/crypto +// hash or a wider digest is unwarranted — deliberately not done. type evaluationAggregationKey struct { flagKey string variant string @@ -145,6 +125,30 @@ type evaluationEntry struct { errorMessage string } +// observe records one more evaluation against an existing bucket: it bumps the count and +// widens the [firstEvaluation, lastEvaluation] window to include nowMs. Every existing-bucket +// path across the three tiers funnels through here so the count++/min/max logic lives once. +func (e *evaluationEntry) observe(nowMs int64) { + e.count++ + if nowMs < e.firstEvaluation { + e.firstEvaluation = nowMs + } + if nowMs > e.lastEvaluation { + e.lastEvaluation = nowMs + } +} + +// newEvaluationEntry returns a fresh bucket for nowMs with count 1 and first==last==nowMs. +// Callers set any tier-specific fields (runtimeDefault, targetingKey, contextAttrs, +// errorMessage) on the returned entry. +func newEvaluationEntry(nowMs int64) *evaluationEntry { + return &evaluationEntry{ + count: 1, + firstEvaluation: nowMs, + lastEvaluation: nowMs, + } +} + // flagEvaluationAggregator holds the three-tier aggregation maps. type flagEvaluationAggregator struct { mu sync.Mutex @@ -393,17 +397,12 @@ func (w *flagEvaluationWriter) flush() { nowMs := time.Now().UnixMilli() var events []flagEvaluationEvent - // Full tier events. + // Full tier: required fields + variant + allocation + targeting_key + context + error; + // runtime_default decorates this tier. for key, e := range full { - ev := flagEvaluationEvent{ - Timestamp: nowMs, - Flag: flagEvalFlag{Key: key.flagKey}, - FirstEvaluation: e.firstEvaluation, - LastEvaluation: e.lastEvaluation, - EvaluationCount: e.count, - RuntimeDefault: e.runtimeDefault, - TargetingKey: e.targetingKey, - } + ev := baseFlagEvaluationEvent(key.flagKey, e, nowMs) + ev.RuntimeDefault = e.runtimeDefault + ev.TargetingKey = e.targetingKey if key.variant != "" { ev.Variant = &flagEvalVariant{Key: key.variant} } @@ -419,16 +418,11 @@ func (w *flagEvaluationWriter) flush() { events = append(events, ev) } - // Degraded tier events (no targeting_key, no context.evaluation). + // Degraded tier: required fields + variant + allocation; no targeting_key, no context. + // runtime_default decorates this tier. for key, e := range degraded { - ev := flagEvaluationEvent{ - Timestamp: nowMs, - Flag: flagEvalFlag{Key: key.flagKey}, - FirstEvaluation: e.firstEvaluation, - LastEvaluation: e.lastEvaluation, - EvaluationCount: e.count, - RuntimeDefault: e.runtimeDefault, - } + ev := baseFlagEvaluationEvent(key.flagKey, e, nowMs) + ev.RuntimeDefault = e.runtimeDefault if key.variant != "" { ev.Variant = &flagEvalVariant{Key: key.variant} } @@ -438,15 +432,9 @@ func (w *flagEvaluationWriter) flush() { events = append(events, ev) } - // Ultra-degraded tier events (only required fields + flag + variant). + // Ultra-degraded tier: required fields + flag + variant only. Never sets runtime_default. for key, e := range ultraDeg { - ev := flagEvaluationEvent{ - Timestamp: nowMs, - Flag: flagEvalFlag{Key: key.flagKey}, - FirstEvaluation: e.firstEvaluation, - LastEvaluation: e.lastEvaluation, - EvaluationCount: e.count, - } + ev := baseFlagEvaluationEvent(key.flagKey, e, nowMs) if key.variant != "" { ev.Variant = &flagEvalVariant{Key: key.variant} } @@ -469,6 +457,20 @@ func (w *flagEvaluationWriter) flush() { } } +// baseFlagEvaluationEvent builds a flagEvaluationEvent with ONLY the five required schema +// fields (timestamp, flag.key, first/last evaluation, evaluation_count). It is tier-agnostic +// and sets no optional field — RuntimeDefault and the rest are decoration applied by each tier +// loop in flush() after the call. +func baseFlagEvaluationEvent(flagKey string, e *evaluationEntry, nowMs int64) flagEvaluationEvent { + return flagEvaluationEvent{ + Timestamp: nowMs, + Flag: flagEvalFlag{Key: flagKey}, + FirstEvaluation: e.firstEvaluation, + LastEvaluation: e.lastEvaluation, + EvaluationCount: e.count, + } +} + // record runs on the evaluation hot path (the Finally hook). It does only cheap scalar // extraction plus the SDK's shallow context copy, then a non-blocking enqueue — no // flatten/prune/hash/aggregation happens here; the background worker does that. If the @@ -578,22 +580,11 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an contextHash: hashContext(contextAttrs), } - // Fast path: this exact full-tier bucket already exists → increment its count. - // - // This branch is ALSO where a contextHash collision lands (see the - // evaluationAggregationKey doc): if two evaluations share all five exact dimensions and - // their differing contexts hash equal, the second matches here and merges into the - // first's bucket. The count is preserved — never dropped, never misattributed; only the - // second context's distinct attrs are not separately recorded. This is the - // count-preserving guarantee TestSaturationCountPreservation asserts. + // Fast path: this exact full-tier bucket already exists → increment its count. This is + // also where a contextHash collision lands, merging count-preservingly into the existing + // bucket (see the evaluationAggregationKey doc for the collision analysis). if e, ok := a.full[fullKey]; ok { - e.count++ - if nowMs < e.firstEvaluation { - e.firstEvaluation = nowMs - } - if nowMs > e.lastEvaluation { - e.lastEvaluation = nowMs - } + e.observe(nowMs) return } @@ -644,13 +635,7 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { } if e, ok := a.degraded[degKey]; ok { - e.count++ - if nowMs < e.firstEvaluation { - e.firstEvaluation = nowMs - } - if nowMs > e.lastEvaluation { - e.lastEvaluation = nowMs - } + e.observe(nowMs) return } @@ -661,12 +646,9 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { return } - a.degraded[degKey] = &evaluationEntry{ - count: 1, - firstEvaluation: nowMs, - lastEvaluation: nowMs, - runtimeDefault: d.runtimeDefault, - } + e := newEvaluationEntry(nowMs) + e.runtimeDefault = d.runtimeDefault + a.degraded[degKey] = e } // addToUltraDegraded adds an entry to the ultra-degraded map (only flag key + variant). @@ -686,50 +668,30 @@ func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64 } if e, ok := a.ultraDeg[ultraKey]; ok { - e.count++ - if nowMs < e.firstEvaluation { - e.firstEvaluation = nowMs - } - if nowMs > e.lastEvaluation { - e.lastEvaluation = nowMs - } + e.observe(nowMs) return } // New ultra-degraded bucket — enforce the explicit cap. Resolve the effective cap with // the zero-value footgun policy: a non-positive ultraDegCap means "use the default" so a // forgotten field does not silently route everything to the sentinel. - cap := a.ultraDegCap - if cap <= 0 { - cap = defaultEvalUltraDegradedCap + effectiveCap := a.ultraDegCap + if effectiveCap <= 0 { + effectiveCap = defaultEvalUltraDegradedCap } - if len(a.ultraDeg) >= cap { + if len(a.ultraDeg) >= effectiveCap { // Cap reached — collapse this and all further over-cap distinct keys into a single // sentinel overflow bucket so the count is preserved without unbounded growth. overflowKey := evaluationUltraDegradedKey{flagKey: ultraDegradedOverflowFlagKey} if e, ok := a.ultraDeg[overflowKey]; ok { - e.count++ - if nowMs < e.firstEvaluation { - e.firstEvaluation = nowMs - } - if nowMs > e.lastEvaluation { - e.lastEvaluation = nowMs - } + e.observe(nowMs) return } - a.ultraDeg[overflowKey] = &evaluationEntry{ - count: 1, - firstEvaluation: nowMs, - lastEvaluation: nowMs, - } + a.ultraDeg[overflowKey] = newEvaluationEntry(nowMs) return } - a.ultraDeg[ultraKey] = &evaluationEntry{ - count: 1, - firstEvaluation: nowMs, - lastEvaluation: nowMs, - } + a.ultraDeg[ultraKey] = newEvaluationEntry(nowMs) } // context value type discriminators for the canonical hash encoding. Each distinct Go type @@ -745,20 +707,14 @@ const ( ctxTagOther byte = 'o' ) -// hashContext computes a deterministic 64-bit FNV-1a hash of the pruned context map for -// use as a discriminator inside evaluationAggregationKey. The enumerable struct fields -// (flagKey, variant, allocationKey, reason, targetingKey) are exact string comparisons and -// cannot collide. +// hashContext computes a deterministic 64-bit FNV-1a hash of the pruned context map, used as +// the contextHash discriminator inside evaluationAggregationKey. // -// The per-field encoding is CANONICAL: each field is written as a length-delimited key -// (fixed-width 8-byte big-endian length + key bytes) followed by a type-tag byte and a -// length-delimited rendered value. Because both key and value are length-delimited and the -// value is type-tagged, distinct contexts cannot ALIAS by construction: int 1 and string -// "1" carry different tags, and '=' / '\n' embedded in a key or value can no longer be -// mistaken for a field delimiter. A digest collision is therefore now a genuine 64-bit hash -// collision, not a systematic encoding alias. A collision is still count-preserving (it -// merges into an existing bucket via add()'s fast path, costing only context-attribute -// fidelity, never a count) — see the evaluationAggregationKey doc for the full analysis. +// The per-field encoding is CANONICAL — each field is a length-delimited key followed by a +// type-tag byte and a length-delimited value — so distinct contexts cannot ALIAS by +// construction (int 1 vs string "1" differ by tag; '=' / '\n' cannot fake a field boundary). +// Any digest collision is thus a genuine 64-bit hash collision, which is count-preserving (see +// the evaluationAggregationKey doc). func hashContext(attrs map[string]any) uint64 { if len(attrs) == 0 { return 0 From 0e12a5c00bfda4bd76d82fda12bfff8d4113da32 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 10:12:05 -0400 Subject: [PATCH 17/37] feat(openfeature): 2-tier EVP aggregation (full -> degraded -> drop), remove ultra-degraded; resize caps for 2,500-flag scale --- openfeature/flagevaluation.go | 156 +++++++++--------------- openfeature/flagevaluation_hook_test.go | 5 +- openfeature/flagevaluation_test.go | 113 ++++++++--------- 3 files changed, 109 insertions(+), 165 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 2e8fbb500a7..1e2e358d16b 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -47,20 +47,19 @@ const ( maxFieldLength = 256 // Aggregation caps. - defaultEvalGlobalCap = 65_536 // bounds full-tier buckets only; degraded/ultra are bounded separately - defaultEvalPerFlagCap = 10_000 // bounds full-fidelity buckets per flag - defaultEvalDegradedCap = 10_000 // bounds degraded map - - // defaultEvalUltraDegradedCap bounds the number of distinct ultra-degraded buckets. - // The ultra-degraded key is (flagKey, variant) and the flag key is caller-supplied, so a - // dynamic/malicious flag key would otherwise grow this map without bound. Distinct keys - // beyond the cap collapse into a single count-preserving sentinel overflow bucket. - defaultEvalUltraDegradedCap = 10_000 - - // ultraDegradedOverflowFlagKey is the reserved flag key for the single sentinel bucket - // that absorbs counts for all over-cap distinct ultra-degraded keys, so the total - // evaluation count is preserved even when the ultra-degraded cardinality is capped. - ultraDegradedOverflowFlagKey = "__other__" + // + // SPIKE cq7 (2-tier resize): the cascade is now full → degraded → drop(counted). With no + // ultra-degraded backstop, degradedCap must hold the FULL legitimate degraded cardinality at + // the team's >=2,500-flag scale target, or legitimate counts would be dropped. The measured + // legitimate degraded cardinality at 2,500 flags (Σ variants×allocations×reasons over a + // realistic flag mix) is 21,662 distinct buckets (see flagevaluation_scale_test.go); so + // degradedCap is raised to 32,768 (~1.5x headroom). globalCap (full-tier) is raised to + // 131,072 so a realistic 2,500-flag × multi-context workload keeps full-fidelity buckets + // before degrading rather than dropping them — overflow from full still cascades to degraded, + // which is now sized to hold it. + defaultEvalGlobalCap = 131_072 // bounds full-tier buckets only; degraded is bounded separately + defaultEvalPerFlagCap = 10_000 // bounds full-fidelity buckets per flag + defaultEvalDegradedCap = 32_768 // bounds degraded map; overflow is dropped(counted), no ultra tier // defaultEvalEventBufferSize bounds the async hand-off queue between the (hot-path) // Finally hook and the background aggregation worker. On overflow the hook drops the @@ -83,7 +82,7 @@ const ( // the second evaluation merges into the existing bucket via add()'s fast path. This is // count-preserving (Σ(counts across tiers) == add() calls — TestSaturationCountPreservation) // and never misattributing (the exact dimensions still match); the sole casualty is -// context-attribute fidelity, which the degraded/ultra tiers already drop by design. This is +// context-attribute fidelity, which the degraded tier already drops by design. This is // internal telemetry over the customer's own context, not a trust boundary, so a keyed/crypto // hash or a wider digest is unwarranted — deliberately not done. type evaluationAggregationKey struct { @@ -95,8 +94,11 @@ type evaluationAggregationKey struct { contextHash uint64 // context is map[string]any (not comparable); hash is a discriminator only } -// evaluationDegradedKey is the key for the degraded aggregation map. -// Drops targeting key, context, and targeting rule key relative to the full key. +// evaluationDegradedKey is the key for the degraded aggregation map — the terminal aggregation +// tier in the 2-tier design (full → degraded → drop). Drops targeting key, context, and +// targeting rule key relative to the full key. This is EXACTLY the OTel feature_flag.evaluations +// cardinality. When a NEW degraded bucket would exceed degradedCap, the count is dropped and +// counted (aggregator.dropped) rather than cascading to a further-degraded tier. type evaluationDegradedKey struct { flagKey string variant string @@ -104,15 +106,6 @@ type evaluationDegradedKey struct { reason string } -// evaluationUltraDegradedKey is the key for the ultra-degraded aggregation map. -// Contains only flag key + variant. Because flagKey is caller-supplied, the distinct -// cardinality of this map is bounded EXPLICITLY by ultraDegCap (see addToUltraDegraded); -// over-cap distinct keys collapse into a single count-preserving sentinel bucket. -type evaluationUltraDegradedKey struct { - flagKey string - variant string -} - // evaluationEntry holds per-window counts and time bounds for one aggregation bucket. type evaluationEntry struct { count int64 @@ -149,22 +142,25 @@ func newEvaluationEntry(nowMs int64) *evaluationEntry { } } -// flagEvaluationAggregator holds the three-tier aggregation maps. +// flagEvaluationAggregator holds the two-tier aggregation maps (full → degraded → drop). type flagEvaluationAggregator struct { mu sync.Mutex full map[evaluationAggregationKey]*evaluationEntry degraded map[evaluationDegradedKey]*evaluationEntry - ultraDeg map[evaluationUltraDegradedKey]*evaluationEntry perFlagFull map[string]int // flagKey → count of full-fidelity entries for this flag globalCount int globalCap int perFlagCap int degradedCap int - ultraDegCap int // bounds distinct ultra-degraded buckets; <=0 is treated as the default + // dropped counts evaluations whose count was lost because a NEW degraded bucket would have + // exceeded degradedCap (the terminal tier in the 2-tier design). It is the observable signal + // that legitimate counts are being dropped and that degradedCap should be raised. It is + // distinct from flagEvaluationWriter.dropped (which counts async-queue backpressure drops). + droppedDegradedOverflow int64 } // flagEvaluationEvent matches flagevaluation.json — required fields always present; -// optional fields use omitempty (absent = schema-valid for degraded/ultra-degraded tiers). +// optional fields use omitempty (absent = schema-valid for the degraded tier). type flagEvaluationEvent struct { Timestamp int64 `json:"timestamp"` Flag flagEvalFlag `json:"flag"` @@ -298,12 +294,10 @@ func newFlagEvaluationWriter(config ProviderConfig) *flagEvaluationWriter { aggregator: flagEvaluationAggregator{ full: make(map[evaluationAggregationKey]*evaluationEntry), degraded: make(map[evaluationDegradedKey]*evaluationEntry), - ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), perFlagFull: make(map[string]int), globalCap: defaultEvalGlobalCap, perFlagCap: defaultEvalPerFlagCap, degradedCap: defaultEvalDegradedCap, - ultraDegCap: defaultEvalUltraDegradedCap, }, } } @@ -375,25 +369,36 @@ func (w *flagEvaluationWriter) flush() { w.aggregator.mu.Lock() - // Under lock: drain all three maps. + // Under lock: drain both maps. full := w.aggregator.full degraded := w.aggregator.degraded - ultraDeg := w.aggregator.ultraDeg - if (len(full) + len(degraded) + len(ultraDeg)) == 0 { + // Surface degraded-overflow drops (the terminal-tier backstop in the 2-tier design) so an + // undersized degradedCap is observable rather than a silent loss of legitimate counts. + degradedOverflow := w.aggregator.droppedDegradedOverflow + + if (len(full) + len(degraded)) == 0 { + w.aggregator.droppedDegradedOverflow = 0 w.aggregator.mu.Unlock() + if degradedOverflow > 0 { + log.Warn("openfeature: degraded aggregation tier full — dropped %d evaluation(s); raise degradedCap (best-effort telemetry)", degradedOverflow) + } return } // Reset maps. w.aggregator.full = make(map[evaluationAggregationKey]*evaluationEntry) w.aggregator.degraded = make(map[evaluationDegradedKey]*evaluationEntry) - w.aggregator.ultraDeg = make(map[evaluationUltraDegradedKey]*evaluationEntry) w.aggregator.perFlagFull = make(map[string]int) w.aggregator.globalCount = 0 + w.aggregator.droppedDegradedOverflow = 0 w.aggregator.mu.Unlock() + if degradedOverflow > 0 { + log.Warn("openfeature: degraded aggregation tier full — dropped %d evaluation(s); raise degradedCap (best-effort telemetry)", degradedOverflow) + } + nowMs := time.Now().UnixMilli() var events []flagEvaluationEvent @@ -432,15 +437,6 @@ func (w *flagEvaluationWriter) flush() { events = append(events, ev) } - // Ultra-degraded tier: required fields + flag + variant only. Never sets runtime_default. - for key, e := range ultraDeg { - ev := baseFlagEvaluationEvent(key.flagKey, e, nowMs) - if key.variant != "" { - ev.Variant = &flagEvalVariant{Key: key.variant} - } - events = append(events, ev) - } - if len(events) == 0 { return } @@ -559,7 +555,7 @@ func (w *flagEvaluationWriter) sendToAgent(payload flagEvaluationPayload) error // add records one evaluation observation into the appropriate aggregation tier. // Must be called WITHOUT the aggregator lock held (it acquires the lock internally). -// Implements the three-tier cascade: full → degraded → ultra-degraded. +// Implements the two-tier cascade: full → degraded → drop(counted). // // Per-flag attempt counting: perFlagFull[flag] is incremented on every call for a flag // (whether or not a full-tier bucket is actually created). This ensures that once @@ -602,11 +598,12 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an // Check globalCap before creating a new full-tier bucket. if a.globalCount >= a.globalCap { - // Global cap full — count must not be lost. - // Route into ultra-degraded (flag key + counts only) so the count signal is preserved. - // The per-flag attempt counter was already incremented above; once it reaches - // perFlagCap this flag will route through addToDegraded instead. - a.addToUltraDegraded(d, nowMs) + // Global full-tier cap full — count must not be lost. Route into the degraded tier + // (which drops targeting_key + context), sized to hold the legitimate degraded + // cardinality at the >=2,500-flag scale target. The per-flag attempt counter was + // already incremented above; once it reaches perFlagCap this flag routes through + // addToDegraded directly as well. + a.addToDegraded(d, nowMs) return } @@ -624,8 +621,13 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an } // addToDegraded adds an entry to the degraded map (drops targeting_key + context). -// Called with the aggregator lock held. -// The degraded map is capped by degradedCap; overflow routes to ultra-degraded. +// Called with the aggregator lock held. Degraded is the TERMINAL aggregation tier in the +// 2-tier design: when a NEW degraded bucket would exceed degradedCap, the evaluation's count is +// DROPPED and counted (droppedDegradedOverflow) rather than cascading to a further-degraded +// tier. degradedCap is sized (defaultEvalDegradedCap) to hold the legitimate degraded +// cardinality at the >=2,500-flag scale target, so this drop only fires under cardinality far +// beyond that target (e.g. an unbounded dynamic/abusive flag key — reviewer concern #8) and the +// dropped counter makes such overflow observable. func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { degKey := evaluationDegradedKey{ flagKey: d.flagKey, @@ -641,8 +643,9 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { // New degraded bucket — check degradedCap. if len(a.degraded) >= a.degradedCap { - // degradedCap exceeded — fall through to ultra-degraded. - a.addToUltraDegraded(d, nowMs) + // degradedCap exceeded — terminal tier full. Drop the count but keep it observable so an + // undersized cap surfaces in the flush warning instead of silently losing legitimate data. + a.droppedDegradedOverflow++ return } @@ -651,49 +654,6 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { a.degraded[degKey] = e } -// addToUltraDegraded adds an entry to the ultra-degraded map (only flag key + variant). -// Called with the aggregator lock held. -// -// The ultra-degraded tier is EXPLICITLY capped by ultraDegCap (with <=0 treated as -// defaultEvalUltraDegradedCap). Because flagKey is caller-supplied, an unbounded number of -// dynamic/malicious flag keys would otherwise grow this map without bound (reviewer concern -// #8). When creating a NEW bucket would exceed the effective cap, the count is routed into a -// single count-preserving sentinel overflow bucket (flag key ultraDegradedOverflowFlagKey, -// empty variant) instead. Existing buckets always increment normally, so the total -// evaluation count is preserved (Σ across tiers == add() calls). -func (a *flagEvaluationAggregator) addToUltraDegraded(d evalDetails, nowMs int64) { - ultraKey := evaluationUltraDegradedKey{ - flagKey: d.flagKey, - variant: d.variant, - } - - if e, ok := a.ultraDeg[ultraKey]; ok { - e.observe(nowMs) - return - } - - // New ultra-degraded bucket — enforce the explicit cap. Resolve the effective cap with - // the zero-value footgun policy: a non-positive ultraDegCap means "use the default" so a - // forgotten field does not silently route everything to the sentinel. - effectiveCap := a.ultraDegCap - if effectiveCap <= 0 { - effectiveCap = defaultEvalUltraDegradedCap - } - if len(a.ultraDeg) >= effectiveCap { - // Cap reached — collapse this and all further over-cap distinct keys into a single - // sentinel overflow bucket so the count is preserved without unbounded growth. - overflowKey := evaluationUltraDegradedKey{flagKey: ultraDegradedOverflowFlagKey} - if e, ok := a.ultraDeg[overflowKey]; ok { - e.observe(nowMs) - return - } - a.ultraDeg[overflowKey] = newEvaluationEntry(nowMs) - return - } - - a.ultraDeg[ultraKey] = newEvaluationEntry(nowMs) -} - // context value type discriminators for the canonical hash encoding. Each distinct Go type // gets a distinct tag byte so that, e.g., int 1 and string "1" cannot render identically. const ( diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index 8e3c42c3ff1..e170aadcdac 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -26,7 +26,6 @@ func setupTestWriter(t *testing.T) *flagEvaluationWriter { aggregator: flagEvaluationAggregator{ full: make(map[evaluationAggregationKey]*evaluationEntry), degraded: make(map[evaluationDegradedKey]*evaluationEntry), - ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), perFlagFull: make(map[string]int), globalCap: 10, perFlagCap: 3, @@ -203,7 +202,7 @@ func TestFlagEvaluationHookFinally(t *testing.T) { w.aggregator.mu.Lock() defer w.aggregator.mu.Unlock() - total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) + total := len(w.aggregator.full) + len(w.aggregator.degraded) if total == 0 { t.Error("expected Finally to record an entry, got none") } @@ -240,7 +239,7 @@ func TestFlagEvaluationHookContextCancelled(t *testing.T) { w.aggregator.mu.Lock() defer w.aggregator.mu.Unlock() - total := len(w.aggregator.full) + len(w.aggregator.degraded) + len(w.aggregator.ultraDeg) + total := len(w.aggregator.full) + len(w.aggregator.degraded) if total != 1 { t.Errorf("expected the cancelled-context evaluation to still be counted, got %d entries", total) } diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index c539c91e6aa..18ccbff15ea 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -23,12 +23,10 @@ func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { return &flagEvaluationAggregator{ full: make(map[evaluationAggregationKey]*evaluationEntry), degraded: make(map[evaluationDegradedKey]*evaluationEntry), - ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), perFlagFull: make(map[string]int), globalCap: 10, // small cap to trigger overflow in tests perFlagCap: 3, degradedCap: 3, - ultraDegCap: defaultEvalUltraDegradedCap, // real default keeps the multi-tier cascade from collapsing into the sentinel } } @@ -36,16 +34,14 @@ func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { // Unlike setupTestAggregator (which fixes small caps), each cap is a parameter so a test can // drive a specific tier-overflow scenario. The cap NUMBERS are load-bearing — callers pass // the exact values their scenario requires. -func newTestAggregator(globalCap, perFlagCap, degradedCap, ultraDegCap int) *flagEvaluationAggregator { +func newTestAggregator(globalCap, perFlagCap, degradedCap int) *flagEvaluationAggregator { return &flagEvaluationAggregator{ full: make(map[evaluationAggregationKey]*evaluationEntry), degraded: make(map[evaluationDegradedKey]*evaluationEntry), - ultraDeg: make(map[evaluationUltraDegradedKey]*evaluationEntry), perFlagFull: make(map[string]int), globalCap: globalCap, perFlagCap: perFlagCap, degradedCap: degradedCap, - ultraDegCap: ultraDegCap, } } @@ -179,8 +175,10 @@ func TestFlagEvaluationPayloadSchema(t *testing.T) { optionalAbsent: []string{"targeting_key", "context"}, }, { - name: "ultra-degraded tier has only required fields", - // Ultra-degraded: only flag key + counts; no variant, allocation, targeting, context. + name: "required-only event omits all optional fields", + // A bare event carrying only flag key + counts; no variant, allocation, targeting, + // context. (This shape is no longer emitted by an ultra-degraded tier, but the + // schema must still accept a required-fields-only event.) event: flagEvaluationEvent{ Timestamp: nowMs, Flag: flagEvalFlag{Key: "test-flag"}, @@ -322,7 +320,7 @@ func TestAggregatorCollisionSafety(t *testing.T) { // Must be run with -race to satisfy the race-free requirement. func TestAggregatorConcurrentMinMax(t *testing.T) { // Caps large enough not to overflow during this test. - agg := newTestAggregator(100_000, 100_000, 100_000, defaultEvalUltraDegradedCap) + agg := newTestAggregator(100_000, 100_000, 100_000) d := evalDetails{ flagKey: "concurrent-flag", @@ -361,31 +359,21 @@ func TestAggregatorConcurrentMinMax(t *testing.T) { } } -// TestSaturationCountPreservation is the regression guard against a silent drop at -// globalCap overflow. It asserts that the sum of all evaluation counts across ALL tiers -// (full + degraded + ultra-degraded) equals the total number of add() calls, even after -// BOTH globalCap AND perFlagCap have been exhausted. -// -// This test MUST FAIL on the pre-fix code (negative control proving the silent drop), and -// MUST PASS after rerouting the globalCap-overflow return into the ultra-degraded tier. +// TestSaturationCountPreservation is the regression guard against a SILENT drop at saturation +// in the 2-tier design. Because the degraded tier is now terminal (overflow is dropped, not +// cascaded), the invariant is: Σ(full+degraded counts) + droppedDegradedOverflow == add() calls. +// No evaluation may vanish without being COUNTED — silent loss is the defect this guards against. func TestSaturationCountPreservation(t *testing.T) { // Use small caps so we can saturate them quickly. // globalCap=5 means only 5 full-tier buckets ever created. // perFlagCap=2 means after 2 distinct full-tier buckets per flag, it overflows to degraded. - // degradedCap=3 means only 3 degraded buckets; further overflow goes to ultra-degraded. - agg := newTestAggregator(5, 2, 3, defaultEvalUltraDegradedCap) + // degradedCap=3 means only 3 degraded buckets; further overflow is dropped(counted). + agg := newTestAggregator(5, 2, 3) nowMs := time.Now().UnixMilli() - // We drive 100 distinct evaluations. Each call to add() must contribute exactly 1 - // count unit to one of the three tiers. After all calls, the Σ must equal 100. - // - // Strategy: use 20 different flag keys × 5 distinct allocationKey combos so that: - // - The first 2 per flag go into the full tier (perFlagCap=2). - // - The next ones overflow to degraded (bounded by degradedCap=3). - // - Once degraded is also full, overflow to ultra-degraded. - // - Once globalCap(5) is hit, any flag's attempt-count not yet at perFlagCap routes - // through the globalCap branch — the defect being guarded against is that this branch - // returns silently instead of routing to ultra-degraded. + // Drive 100 distinct evaluations. Each add() must contribute exactly 1 count unit to either + // the full tier, the degraded tier, or the droppedDegradedOverflow counter. After all calls, + // Σ(full+degraded) + dropped must equal 100 — nothing silently lost. const totalCalls = 100 for i := range totalCalls { flagIdx := i % 20 @@ -400,7 +388,7 @@ func TestSaturationCountPreservation(t *testing.T) { agg.add(d, nil, nowMs) } - // Sum counts across all three tiers. + // Sum counts across both tiers plus the observable drop counter. agg.mu.Lock() defer agg.mu.Unlock() @@ -411,16 +399,14 @@ func TestSaturationCountPreservation(t *testing.T) { for _, e := range agg.degraded { totalCounted += e.count } - for _, e := range agg.ultraDeg { - totalCounted += e.count - } + totalCounted += agg.droppedDegradedOverflow if totalCounted != totalCalls { t.Errorf( - "count preservation violated: Σ(full+degraded+ultraDeg)=%d, expected=%d (add() calls); "+ - "silent drops detected (full buckets=%d, degraded buckets=%d, ultraDeg buckets=%d)", + "count preservation violated: Σ(full+degraded)+dropped=%d, expected=%d (add() calls); "+ + "silent drops detected (full buckets=%d, degraded buckets=%d, droppedDegradedOverflow=%d)", totalCounted, totalCalls, - len(agg.full), len(agg.degraded), len(agg.ultraDeg), + len(agg.full), len(agg.degraded), agg.droppedDegradedOverflow, ) } } @@ -470,7 +456,7 @@ func TestAggregatorCapOverflow(t *testing.T) { } }) - t.Run("degradedCap overflow routes to ultra-degraded", func(t *testing.T) { + t.Run("degradedCap overflow is dropped and counted", func(t *testing.T) { agg := setupTestAggregator(t) // degradedCap=3 nowMs := time.Now().UnixMilli() @@ -489,8 +475,8 @@ func TestAggregatorCapOverflow(t *testing.T) { } } - // Continue adding until degradedCap is also exhausted. - // At that point, ultra-degraded must receive new entries. + // Continue adding until degradedCap is also exhausted. At that point — with no ultra + // tier — new degraded buckets must be dropped and COUNTED (droppedDegradedOverflow). for i := range 10 { d := evalDetails{ flagKey: fmt.Sprintf("overflow-flag-%d", i), @@ -512,8 +498,11 @@ func TestAggregatorCapOverflow(t *testing.T) { agg.mu.Lock() defer agg.mu.Unlock() - if len(agg.ultraDeg) == 0 { - t.Error("expected ultra-degraded buckets after degradedCap overflow") + if len(agg.degraded) > agg.degradedCap { + t.Errorf("degraded tier %d exceeds degradedCap %d — terminal tier not bounded", len(agg.degraded), agg.degradedCap) + } + if agg.droppedDegradedOverflow == 0 { + t.Error("expected droppedDegradedOverflow > 0 after degradedCap exhaustion (drops must be counted, not silent)") } }) @@ -522,10 +511,9 @@ func TestAggregatorCapOverflow(t *testing.T) { nowMs := time.Now().UnixMilli() // Add 50 distinct evaluations (each a unique flag key). - // globalCap=10 caps the full tier; overflow goes to ultra-degraded. - // The full tier must stay at or below globalCap. - // The total count across all tiers must equal the number of add() calls - // (no silent drops). + // globalCap=10 caps the full tier; overflow cascades to degraded (then drops if degraded + // is also full). The full tier must stay at or below globalCap, and the total count + // across both tiers plus the drop counter must equal the number of add() calls. const calls = 50 for i := range calls { d := evalDetails{ @@ -547,7 +535,7 @@ func TestAggregatorCapOverflow(t *testing.T) { t.Errorf("full-tier buckets %d exceeds globalCap %d", len(agg.full), agg.globalCap) } - // Every add() call must have produced a count unit in some tier (no silent drops). + // Every add() call must have produced a count unit somewhere observable (no silent drops). var totalCounted int64 for _, e := range agg.full { totalCounted += e.count @@ -555,11 +543,9 @@ func TestAggregatorCapOverflow(t *testing.T) { for _, e := range agg.degraded { totalCounted += e.count } - for _, e := range agg.ultraDeg { - totalCounted += e.count - } + totalCounted += agg.droppedDegradedOverflow if totalCounted != calls { - t.Errorf("count preservation violated: Σ(all tiers)=%d, expected=%d", totalCounted, calls) + t.Errorf("count preservation violated: Σ(full+degraded)+dropped=%d, expected=%d", totalCounted, calls) } }) } @@ -678,14 +664,15 @@ func TestHashContextCanonicalEncoding(t *testing.T) { }) } -// TestUltraDegradedCapBounded guards Finding #3 (ultra-degraded tier unbounded). With a -// small positive ultraDegCap, many distinct flag keys must NOT grow ultraDeg without bound: -// len(ultraDeg) <= ultraDegCap+1 (the +1 is the sentinel overflow bucket) and counts are -// preserved (Σ across tiers == add() calls). -func TestUltraDegradedCapBounded(t *testing.T) { +// TestDegradedCapBounded guards reviewer concern #8 (unbounded dynamic/abusive flag keys) under +// the 2-tier design. With the degraded tier as the terminal tier, an unbounded number of distinct +// flag keys must NOT grow the degraded map without bound: len(degraded) <= degradedCap, and the +// over-cap counts must be DROPPED-AND-COUNTED (droppedDegradedOverflow), never silently lost. +// Σ(full+degraded counts) + droppedDegradedOverflow must equal the add() call count. +func TestDegradedCapBounded(t *testing.T) { const cap = 3 - // globalCap=0 and degradedCap=0 force everything straight into the ultra-degraded tier. - agg := newTestAggregator(0, 100_000, 0, cap) + // globalCap=0 forces every distinct full key straight past the full tier into degraded. + agg := newTestAggregator(0, 100_000, cap) nowMs := time.Now().UnixMilli() const calls = 100 @@ -701,8 +688,8 @@ func TestUltraDegradedCapBounded(t *testing.T) { agg.mu.Lock() defer agg.mu.Unlock() - if len(agg.ultraDeg) > cap+1 { - t.Errorf("ultra-degraded cardinality %d exceeds ultraDegCap+1 (%d) — tier not bounded", len(agg.ultraDeg), cap+1) + if len(agg.degraded) > cap { + t.Errorf("degraded cardinality %d exceeds degradedCap (%d) — terminal tier not bounded", len(agg.degraded), cap) } var total int64 @@ -712,16 +699,14 @@ func TestUltraDegradedCapBounded(t *testing.T) { for _, e := range agg.degraded { total += e.count } - for _, e := range agg.ultraDeg { - total += e.count - } + total += agg.droppedDegradedOverflow if total != calls { - t.Errorf("count preservation violated under ultraDegCap: Σ(all tiers)=%d, expected=%d", total, calls) + t.Errorf("count preservation violated under degradedCap: Σ(full+degraded)+dropped=%d, expected=%d", total, calls) } - // The sentinel overflow bucket must be present and carry the over-cap counts. - if _, ok := agg.ultraDeg[evaluationUltraDegradedKey{flagKey: ultraDegradedOverflowFlagKey}]; !ok { - t.Errorf("expected sentinel overflow bucket %q in ultra-degraded tier", ultraDegradedOverflowFlagKey) + // The over-cap counts must be observable in the drop counter. + if agg.droppedDegradedOverflow == 0 { + t.Errorf("expected droppedDegradedOverflow > 0 when distinct keys exceed degradedCap") } } From a699b77b9116efd386fb464a968e99405df3ca66 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 10:12:06 -0400 Subject: [PATCH 18/37] test(openfeature): add 2,500-flag scale + parallel benchmarks and aggregator scale harness --- openfeature/flagevaluation_scale_test.go | 370 +++++++++++++++++++++++ openfeature/provider_bench_test.go | 68 +++++ 2 files changed, 438 insertions(+) create mode 100644 openfeature/flagevaluation_scale_test.go diff --git a/openfeature/flagevaluation_scale_test.go b/openfeature/flagevaluation_scale_test.go new file mode 100644 index 00000000000..e5f3d8fe92a --- /dev/null +++ b/openfeature/flagevaluation_scale_test.go @@ -0,0 +1,370 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +package openfeature + +import ( + "fmt" + "testing" + "time" +) + +// SPIKE (cq7): decisive measurement for the "remove the ultra-degraded tier" decision. +// +// These tests drive flagEvaluationAggregator.add directly (no client, no hooks, no async +// worker) to deterministically simulate the team's >=2,500-flag scale target with realistic +// flag structure, then INSPECT the three aggregation maps. The goal is to answer: +// +// 1. At 2,500 flags, under what conditions (if any) does the ultra-degraded tier actually +// trigger? Does it require saturating BOTH full(65k) AND degraded(10k)? +// 2. Does 2,500 flags' worth of LEGITIMATE degraded buckets fit under degradedCap=10,000, or +// does it overflow (cascading to ultra today, DROPPING under a 2-tier design)? +// 3. What degradedCap/globalCap values make a 2-tier design never drop legitimate counts at +// 2,500 flags? +// +// They are not assertions about correct behavior so much as a measurement harness; each test +// logs its numbers via t.Logf and is intended to be read with `go test -run TestScale -v`. + +// scaleFlagShape describes the realistic per-flag structure used to size the degraded tier. +// Degraded key = (flagKey, variant, allocationKey, reason), so the LEGITIMATE degraded +// cardinality of a flag is variants × allocations × reasons-actually-observed. +type scaleFlagShape struct { + variants int // distinct variant keys this flag can return + allocations int // distinct allocation keys this flag can return + reasons []string +} + +// realisticReasons is the natural reason set a healthy server-side flag emits. A flag that +// matches targeting emits TARGETING_MATCH/SPLIT for assigned subjects and DEFAULT for the +// rest, so 2 reasons/flag is the realistic steady state (not all 8 canonical reasons). +var realisticReasons = []string{"targeting_match", "default"} + +// makeScaleFlags builds n flags with a realistic spread of variants/allocations. The spread is +// deterministic (driven by index modulo) so the degraded cardinality is exactly reproducible: +// - ~1/3 of flags: 2 variants, 1 allocation (simple on/off) +// - ~1/3 of flags: 3 variants, 1 allocation (multivariate) +// - ~1/3 of flags: 4 variants, 2 allocations (multivariate + multiple allocations) +func makeScaleFlags(n int) []scaleFlagShape { + flags := make([]scaleFlagShape, n) + for i := range n { + switch i % 3 { + case 0: + flags[i] = scaleFlagShape{variants: 2, allocations: 1, reasons: realisticReasons} + case 1: + flags[i] = scaleFlagShape{variants: 3, allocations: 1, reasons: realisticReasons} + default: + flags[i] = scaleFlagShape{variants: 4, allocations: 2, reasons: realisticReasons} + } + } + return flags +} + +// legitimateDegradedCardinality returns the number of DISTINCT degraded buckets the given flag +// shapes would produce if every (variant × allocation × reason) combination were observed. +// This is the count a 2-tier design must be able to hold without dropping. +func legitimateDegradedCardinality(flags []scaleFlagShape) int { + total := 0 + for _, f := range flags { + total += f.variants * f.allocations * len(f.reasons) + } + return total +} + +// driveScale records evaluations into agg for the given flag shapes, distributing each flag's +// evaluations across its variant/allocation/reason combinations and across numContexts distinct +// evaluation contexts (subjects). evalsPerCombo controls how many subjects hit each combination +// (so counts accumulate). It returns the total number of add() calls made. +// +// The context cardinality knob (numContexts) is what splits the FULL tier: the full key includes +// targetingKey + contextHash, so more distinct subjects => more full buckets => earlier full-tier +// saturation => earlier cascade into degraded (and potentially ultra). +func driveScale(agg *flagEvaluationAggregator, flags []scaleFlagShape, numContexts, evalsPerCombo int) int64 { + nowMs := time.Now().UnixMilli() + var calls int64 + ctxCounter := 0 + for fi, f := range flags { + flagKey := fmt.Sprintf("flag-%05d", fi) + for v := range f.variants { + variant := fmt.Sprintf("v%d", v) + for a := range f.allocations { + alloc := fmt.Sprintf("alloc-%d", a) + for _, reason := range f.reasons { + for e := 0; e < evalsPerCombo; e++ { + // Spread subjects across numContexts distinct targeting keys + contexts. + subj := ctxCounter % numContexts + ctxCounter++ + d := evalDetails{ + flagKey: flagKey, + variant: variant, + allocationKey: alloc, + reason: reason, + targetingKey: fmt.Sprintf("user-%d", subj), + } + attrs := map[string]any{ + "country": fmt.Sprintf("c%d", subj%50), + "plan": fmt.Sprintf("p%d", subj%5), + } + agg.add(d, attrs, nowMs) + calls++ + } + } + } + } + } + return calls +} + +// tierCounts returns observable aggregator state after a run. In the 2-tier design the terminal +// tier is degraded; over-cap counts land in droppedDegradedOverflow (observable, not silent). +// sumCounts includes the drop counter so the count-preservation invariant is Σ == add() calls. +type tierCounts struct { + full, degraded int + dropped int64 + globalCount int + sumCounts int64 +} + +func snapshot(agg *flagEvaluationAggregator) tierCounts { + agg.mu.Lock() + defer agg.mu.Unlock() + tc := tierCounts{ + full: len(agg.full), + degraded: len(agg.degraded), + dropped: agg.droppedDegradedOverflow, + globalCount: agg.globalCount, + } + for _, e := range agg.full { + tc.sumCounts += e.count + } + for _, e := range agg.degraded { + tc.sumCounts += e.count + } + tc.sumCounts += agg.droppedDegradedOverflow + return tc +} + +// TestScaleDegradedCardinality2500Flags reports the LEGITIMATE degraded cardinality for the +// realistic 2,500-flag shape and compares it to the production degradedCap (10,000). This is the +// cap-sizing math that decides whether a 2-tier design drops legitimate counts. +func TestScaleDegradedCardinality2500Flags(t *testing.T) { + const n = 2500 + flags := makeScaleFlags(n) + deg := legitimateDegradedCardinality(flags) + + // Per-shape breakdown for the report. + var s0, s1, s2 int + for i := range n { + switch i % 3 { + case 0: + s0++ + case 1: + s1++ + default: + s2++ + } + } + t.Logf("2,500-flag realistic shape:") + t.Logf(" %d flags @ 2v×1a×2r = %d degraded buckets", s0, s0*2*1*2) + t.Logf(" %d flags @ 3v×1a×2r = %d degraded buckets", s1, s1*3*1*2) + t.Logf(" %d flags @ 4v×2a×2r = %d degraded buckets", s2, s2*4*2*2) + t.Logf("LEGITIMATE degraded cardinality (Σ variants×allocations×reasons) = %d", deg) + t.Logf("production degradedCap = %d", defaultEvalDegradedCap) + t.Logf("production globalCap = %d", defaultEvalGlobalCap) + + if deg > defaultEvalDegradedCap { + t.Logf("RESULT: legitimate degraded cardinality %d EXCEEDS degradedCap %d by %d "+ + "=> under a 2-tier design these would DROP. degradedCap must be raised.", + deg, defaultEvalDegradedCap, deg-defaultEvalDegradedCap) + } else { + t.Logf("RESULT: legitimate degraded cardinality %d FITS under degradedCap %d (headroom %d).", + deg, defaultEvalDegradedCap, defaultEvalDegradedCap-deg) + } + + // Recommendation math: degraded must hold full legitimate cardinality; global (full tier) + // must hold flags × contexts up to a reasonable subject bound. Report a 2x-headroom rec. + recDegraded := roundUpTo(deg*2, 1000) + t.Logf("RECOMMENDATION: degradedCap >= %d (2x headroom over %d legitimate buckets).", recDegraded, deg) +} + +func roundUpTo(v, mult int) int { + if v%mult == 0 { + return v + } + return ((v / mult) + 1) * mult +} + +// TestScaleDropTriggerSweep is the decisive test for the 2-tier design: across a +// context-cardinality sweep at 2,500 flags, it reports whether the terminal-tier DROP +// (droppedDegradedOverflow) ever fires, and under what conditions. It runs each sweep point with +// the (resized) production caps. +func TestScaleDropTriggerSweep(t *testing.T) { + const n = 2500 + flags := makeScaleFlags(n) + deg := legitimateDegradedCardinality(flags) + t.Logf("2,500 flags; legitimate degraded cardinality = %d; production caps "+ + "full=%d perFlag=%d degraded=%d (2-tier: full -> degraded -> drop)", + deg, defaultEvalGlobalCap, defaultEvalPerFlagCap, defaultEvalDegradedCap) + + // Sweep distinct-context cardinality. Low = few subjects (full tier stays small); + // high = many subjects (full tier saturates, cascading to degraded then drop). + sweep := []struct { + name string + numContexts int + evalsPer int + }{ + {"few-contexts (10 subjects)", 10, 2}, + {"moderate-contexts (1k subjects)", 1000, 1}, + {"many-contexts (100k subjects)", 100_000, 1}, + {"extreme-contexts (1M subjects)", 1_000_000, 1}, + } + + for _, sp := range sweep { + t.Run(sp.name, func(t *testing.T) { + agg := newTestAggregator( + defaultEvalGlobalCap, + defaultEvalPerFlagCap, + defaultEvalDegradedCap, + ) + calls := driveScale(agg, flags, sp.numContexts, sp.evalsPer) + tc := snapshot(agg) + + t.Logf("contexts=%d evalsPerCombo=%d => add() calls=%d", sp.numContexts, sp.evalsPer, calls) + t.Logf(" full=%d (globalCount=%d, cap=%d) degraded=%d (cap=%d) droppedDegradedOverflow=%d", + tc.full, tc.globalCount, defaultEvalGlobalCap, + tc.degraded, defaultEvalDegradedCap, tc.dropped) + t.Logf(" Σ counts (full+degraded+dropped)=%d (must == add() calls=%d => %v)", + tc.sumCounts, calls, tc.sumCounts == calls) + + // Count preservation must hold: nothing silently lost. + if tc.sumCounts != calls { + t.Errorf("count preservation violated: Σ=%d != calls=%d", tc.sumCounts, calls) + } + + if tc.dropped > 0 { + t.Logf(" DROP TRIGGERED: %d evaluation(s) dropped (degraded saturated at %d).", + tc.dropped, defaultEvalDegradedCap) + } else { + t.Logf(" DROP NOT TRIGGERED at this sweep point — no legitimate count lost.") + } + }) + } +} + +// TestScaleDropRequiresDegradedSaturation isolates the precondition for a terminal-tier drop: +// the degraded tier must be FULL. With the resized degradedCap and realistic 2,500-flag +// structure, it forces the worst case (globalCap=0, everything cascades to degraded) and reports +// whether legitimate degraded cardinality fits under degradedCap — i.e. whether a 2-tier design +// drops any legitimate counts at the scale target. +func TestScaleDropRequiresDegradedSaturation(t *testing.T) { + const n = 2500 + flags := makeScaleFlags(n) + deg := legitimateDegradedCardinality(flags) + + // Force the full tier to be useless (globalCap=0) so EVERY new bucket cascades immediately + // to the degraded path — the worst case for the degraded tier at 2,500 flags. + agg := newTestAggregator(0, defaultEvalPerFlagCap, defaultEvalDegradedCap) + calls := driveScale(agg, flags, 100_000, 1) + tc := snapshot(agg) + + t.Logf("WORST CASE for degraded tier (globalCap=0 => everything cascades to degraded):") + t.Logf(" legitimate degraded cardinality = %d, degradedCap = %d", deg, defaultEvalDegradedCap) + t.Logf(" result: full=%d degraded=%d droppedDegradedOverflow=%d Σ=%d/calls=%d", + tc.full, tc.degraded, tc.dropped, tc.sumCounts, calls) + + if deg > defaultEvalDegradedCap { + t.Errorf("legitimate degraded cardinality %d EXCEEDS degradedCap %d — 2-tier design would DROP "+ + "legitimate counts at 2,500 flags; raise degradedCap", deg, defaultEvalDegradedCap) + } else { + t.Logf(" => all %d legitimate degraded buckets FIT under degradedCap %d (headroom %d); "+ + "no legitimate count dropped at the scale target.", + deg, defaultEvalDegradedCap, defaultEvalDegradedCap-deg) + if tc.dropped != 0 { + t.Errorf("unexpected drops (%d) when legitimate cardinality fits under degradedCap", tc.dropped) + } + } + + if tc.sumCounts != calls { + t.Errorf("count preservation violated: Σ=%d != calls=%d", tc.sumCounts, calls) + } +} + +// TestScaleFullSaturationCascade saturates the FULL tier naturally (production caps) with +// 2,500 flags × enough distinct subjects that globalCount exceeds globalCap, then reports which +// tier absorbs the overflow. In the 2-tier design, global-cap overflow cascades to degraded +// (which is sized to hold the legitimate degraded cardinality) before any drop. +func TestScaleFullSaturationCascade(t *testing.T) { + const n = 2500 + flags := makeScaleFlags(n) + deg := legitimateDegradedCardinality(flags) + + agg := newTestAggregator( + defaultEvalGlobalCap, + defaultEvalPerFlagCap, + defaultEvalDegradedCap, + ) + // legitimate degraded cardinality is ~21,662 combos; with 8 distinct subjects per combo the + // full tier sees ~173k distinct full keys, over globalCap, forcing overflow into degraded. + calls := driveScale(agg, flags, 1_000_000, 8) + tc := snapshot(agg) + + t.Logf("FULL-saturation cascade (production caps; 8 subjects/combo):") + t.Logf(" legitimate degraded cardinality=%d add() calls=%d", deg, calls) + t.Logf(" full=%d (globalCount=%d, cap=%d) degraded=%d (cap=%d) droppedDegradedOverflow=%d", + tc.full, tc.globalCount, defaultEvalGlobalCap, + tc.degraded, defaultEvalDegradedCap, tc.dropped) + t.Logf(" Σ counts=%d / calls=%d (preserved=%v)", tc.sumCounts, calls, tc.sumCounts == calls) + + if tc.full > defaultEvalGlobalCap { + t.Errorf("full tier %d exceeded globalCap %d", tc.full, defaultEvalGlobalCap) + } + if tc.sumCounts != calls { + t.Errorf("count preservation violated: Σ=%d != calls=%d", tc.sumCounts, calls) + } + t.Logf(" STRUCTURAL NOTE: global-cap overflow cascades to degraded (degraded=%d). "+ + "A drop only occurs once degraded itself reaches degradedCap=%d.", + tc.degraded, defaultEvalDegradedCap) +} + +// TestScaleHotFlagPerFlagCap drives a SINGLE flag past perFlagCap (10,000 distinct full +// buckets) to demonstrate the per-flag overflow path into the degraded tier, and reports whether +// that fill can in turn saturate degradedCap and trigger a terminal-tier drop. +func TestScaleHotFlagPerFlagCap(t *testing.T) { + agg := newTestAggregator( + defaultEvalGlobalCap, + defaultEvalPerFlagCap, + defaultEvalDegradedCap, + ) + nowMs := time.Now().UnixMilli() + + // One hot flag, many distinct (variant, subject) combos so it blows past perFlagCap and then + // keeps generating distinct degraded keys. Degraded key = (flag,variant,alloc,reason); to fill + // the resized degradedCap we need that many distinct (variant,alloc,reason) for this one flag. + const distinctVariants = 50_000 + var calls int64 + for v := 0; v < distinctVariants; v++ { + d := evalDetails{ + flagKey: "hot-flag", + variant: fmt.Sprintf("v%d", v), + allocationKey: "alloc-0", + reason: "targeting_match", + targetingKey: fmt.Sprintf("user-%d", v), + } + agg.add(d, map[string]any{"k": v}, nowMs) + calls++ + } + tc := snapshot(agg) + + t.Logf("HOT-flag perFlagCap path (single flag, %d distinct variants):", distinctVariants) + t.Logf(" full=%d (perFlagCap=%d) degraded=%d (cap=%d) droppedDegradedOverflow=%d", + tc.full, defaultEvalPerFlagCap, tc.degraded, defaultEvalDegradedCap, tc.dropped) + t.Logf(" Σ counts=%d / calls=%d (preserved=%v)", tc.sumCounts, calls, tc.sumCounts == calls) + if tc.dropped > 0 { + t.Logf(" DROP TRIGGERED via a single abusive/hot flag (degraded saturated at %d). "+ + "This is exactly the abuse case the drop counter must make observable.", tc.degraded) + } + if tc.sumCounts != calls { + t.Errorf("count preservation violated: Σ=%d != calls=%d", tc.sumCounts, calls) + } +} diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 7ba4f0b13ac..456c8c0e453 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -276,6 +276,10 @@ type flagEvalBenchProfile struct { var flagEvalBenchProfiles = []flagEvalBenchProfile{ {"typical/100flags_50users_10fields", 100, 50, 10}, {"stress/10flags_1000users_250fields", 10, 1000, 250}, + // scale profile targets the team's >=2,500-flag goal. Flag count is the dimension under + // test, so it dominates; users/fields are kept modest (500/20) so the flag-cardinality + // signal is not swamped by per-evaluation context cost and the suite stays tractable. + {"scale/2500flags_500users_20fields", 2500, 500, 20}, } // runFlagEvalBenchmark drives evaluations through a real OpenFeature client so the provider's @@ -393,3 +397,67 @@ func BenchmarkFlagEvaluationEVPRecord(b *testing.B) { }) } } + +// scaleBenchProfile is the >=2,500-flag profile (the team's scale target) selected from +// flagEvalBenchProfiles by name. It panics if the profile is removed so the scale benches +// fail loudly rather than silently degrade to a smaller config. +func scaleBenchProfile() flagEvalBenchProfile { + const name = "scale/2500flags_500users_20fields" + for _, p := range flagEvalBenchProfiles { + if p.name == name { + return p + } + } + panic("scale profile " + name + " missing from flagEvalBenchProfiles") +} + +// BenchmarkFlagEvaluationOTelPlusEVPParallel measures concurrent server-side evaluation +// through a real OpenFeature client with BOTH the OTel hook and the new EVP flagevaluation +// hook enabled, at the >=2,500-flag scale profile (oleksii review #5). Because every +// concurrent evaluation funnels through the EVP writer's single aggregator mutex +// (flagEvaluationAggregator.add), this is the bench that surfaces lock contention under +// realistic multi-goroutine server load — exactly what the parallel reviewer concern asks for. +// +// Each goroutine rotates its own flag key and targeting key across the full cardinality so the +// aggregator sees a realistic spread of buckets, not a single hot key. The 24h flush interval +// keeps the EVP writer from flushing mid-run so no HTTP round-trip enters the hot path. +// +// Run command: +// +// GOFLAGS=-mod=readonly go test ./openfeature -run='^$' \ +// -bench='^BenchmarkFlagEvaluationOTelPlusEVPParallel$' -benchmem -count=2 -cpu=8 +func BenchmarkFlagEvaluationOTelPlusEVPParallel(b *testing.B) { + p := scaleBenchProfile() + + provider := newDatadogProvider(ProviderConfig{FlagEvaluationFlushInterval: 24 * time.Hour}) + // Disable only the exposure hook; OTel hook + EVP hook + aggregator all run, so this + // measures the combined Path A + Path B cost under contention. + provider.exposureHook = nil + + config := makeBenchmarkConfig(p.numFlags) + provider.updateConfiguration(config) + + initCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + if err := openfeature.SetProviderWithContextAndWait(initCtx, provider); err != nil { + cancel() + b.Fatalf("set provider: %v", err) + } + cancel() + client := openfeature.NewClient("flageval-bench-parallel") + + flagKeys := boolBenchmarkFlagKeys(config) + attrs := makeBenchmarkAttrs(p.numFields) + ctx := context.Background() + + b.ReportAllocs() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + evalCtx := openfeature.NewEvaluationContext("bench-user-"+strconv.Itoa(i%p.numUsers), attrs) + _ = client.Boolean(ctx, flagKeys[i%len(flagKeys)], false, evalCtx) + i++ + } + }) +} From 80824d96d3b6b9e3f16203958557ad01479494f8 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 10:13:45 -0400 Subject: [PATCH 19/37] test(openfeature): modernize range-over-int loops in EVP scale harness (lint) --- openfeature/flagevaluation_scale_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfeature/flagevaluation_scale_test.go b/openfeature/flagevaluation_scale_test.go index e5f3d8fe92a..9fb32f19384 100644 --- a/openfeature/flagevaluation_scale_test.go +++ b/openfeature/flagevaluation_scale_test.go @@ -91,7 +91,7 @@ func driveScale(agg *flagEvaluationAggregator, flags []scaleFlagShape, numContex for a := range f.allocations { alloc := fmt.Sprintf("alloc-%d", a) for _, reason := range f.reasons { - for e := 0; e < evalsPerCombo; e++ { + for range evalsPerCombo { // Spread subjects across numContexts distinct targeting keys + contexts. subj := ctxCounter % numContexts ctxCounter++ @@ -343,7 +343,7 @@ func TestScaleHotFlagPerFlagCap(t *testing.T) { // the resized degradedCap we need that many distinct (variant,alloc,reason) for this one flag. const distinctVariants = 50_000 var calls int64 - for v := 0; v < distinctVariants; v++ { + for v := range distinctVariants { d := evalDetails{ flagKey: "hot-flag", variant: fmt.Sprintf("v%d", v), From 1c648c4db62bb330ea89d49069eb0bab208b6a86 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 11:11:49 -0400 Subject: [PATCH 20/37] fix(openfeature): use comparable canonical-context key for EVP full-tier buckets Address oleksii PR #4886 review #2: replace the lossy FNV-1a contextHash discriminator in evaluationAggregationKey with an exact, comparable canonical-context string (contextKey). The existing type-tagged, length-delimited canonical encoding is now emitted AS THE MAP KEY instead of hashed, so Go's map compares it byte-for-byte. - Distinct contexts now ALWAYS land in distinct full-tier buckets: there is no hash, so no hash collision, so a count can never be misattributed to the wrong context (the prior collision-misattribution case is gone). - Delete hash/fnv import, hashContext, and the collision-analysis doc block. - Rename hashContext -> canonicalContextKey (returns the encoding as a string). - Tests assert distinct contexts (int 1 vs string "1"; '='/newline-bearing values; multi-field) land in DISTINCT buckets with zero misattribution, and identical contexts still merge into one bucket. Memory: the contextKey string is stored once per full-tier bucket, bounded by globalCap and the 256-field/256-char prune. EVPRecord hot path unchanged (the key is built in the async worker, not record()). OTel path untouched. --- openfeature/flagevaluation.go | 85 ++++++++++----------- openfeature/flagevaluation_scale_test.go | 2 +- openfeature/flagevaluation_test.go | 94 +++++++++++++++++------- 3 files changed, 109 insertions(+), 72 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 1e2e358d16b..d412317bb14 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -10,7 +10,6 @@ import ( "cmp" "context" "fmt" - "hash/fnv" "io" "log/slog" "net/http" @@ -67,31 +66,30 @@ const ( defaultEvalEventBufferSize = 4096 ) -// evaluationAggregationKey identifies one full-tier aggregation bucket. Its identity splits -// into two parts with different collision properties: +// evaluationAggregationKey identifies one full-tier aggregation bucket. Every field is an +// EXACT, comparable string compared byte-for-byte by Go map equality, so distinct keys can +// NEVER alias into the same bucket: // -// 1. Enumerable dimensions — flagKey, variant, allocationKey, reason, targetingKey — are -// EXACT string fields compared byte-for-byte by Go map equality, so they can NEVER alias: -// a count is never attributed to the wrong flag/variant/reason/allocation. +// - The enumerable dimensions (flagKey, variant, allocationKey, reason, targetingKey) are +// the raw evaluation strings. +// - contextKey is the EXACT canonical type-tagged, length-delimited encoding of the pruned +// context (see canonicalContextKey). Because it is the full encoding — not a lossy 64-bit +// digest — two distinct contexts ALWAYS produce distinct contextKey strings (oleksii #2): +// int 1 vs string "1" differ by type tag, and '='/'\n'-bearing values cannot fake a field +// boundary thanks to the length prefixes. There is no hash, so there is no hash collision, +// so a count can never be misattributed to the wrong context. Go's map hashes and compares +// the full struct key (including contextKey) natively. // -// 2. contextHash is a 64-bit FNV-1a digest of the pruned context (map[string]any, not -// comparable, so it cannot be a struct field — see hashContext for the canonical encoding -// that makes distinct contexts non-aliasing). -// -// A contextHash collision is therefore a genuine 64-bit hash collision, not an encoding alias: -// the second evaluation merges into the existing bucket via add()'s fast path. This is -// count-preserving (Σ(counts across tiers) == add() calls — TestSaturationCountPreservation) -// and never misattributing (the exact dimensions still match); the sole casualty is -// context-attribute fidelity, which the degraded tier already drops by design. This is -// internal telemetry over the customer's own context, not a trust boundary, so a keyed/crypto -// hash or a wider digest is unwarranted — deliberately not done. +// The contextKey string is stored once per full-tier bucket; its memory footprint is therefore +// bounded by the number of full-tier buckets (globalCap) and the pruned context size +// (256 fields × 256 chars), and measured in BenchmarkFlagEvaluationOTelPlusEVPParallel. type evaluationAggregationKey struct { flagKey string variant string allocationKey string reason string targetingKey string - contextHash uint64 // context is map[string]any (not comparable); hash is a discriminator only + contextKey string // exact canonical encoding of the pruned context; comparable, not a digest } // evaluationDegradedKey is the key for the degraded aggregation map — the terminal aggregation @@ -566,19 +564,20 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an a.mu.Lock() defer a.mu.Unlock() - // Build the full key: exact struct fields for enumerable dims + 64-bit FNV context digest. + // Build the full key: exact, comparable strings for every dimension including the canonical + // context encoding. No hash, so distinct contexts get distinct buckets (oleksii #2). fullKey := evaluationAggregationKey{ flagKey: d.flagKey, variant: d.variant, allocationKey: d.allocationKey, reason: d.reason, targetingKey: d.targetingKey, - contextHash: hashContext(contextAttrs), + contextKey: canonicalContextKey(contextAttrs), } - // Fast path: this exact full-tier bucket already exists → increment its count. This is - // also where a contextHash collision lands, merging count-preservingly into the existing - // bucket (see the evaluationAggregationKey doc for the collision analysis). + // Fast path: this exact full-tier bucket already exists → increment its count. Because + // contextKey is the full canonical encoding (not a digest), this fast path is hit only by a + // genuinely identical pruned context — never by an aliasing collision. if e, ok := a.full[fullKey]; ok { e.observe(nowMs) return @@ -654,7 +653,7 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { a.degraded[degKey] = e } -// context value type discriminators for the canonical hash encoding. Each distinct Go type +// context value type discriminators for the canonical key encoding. Each distinct Go type // gets a distinct tag byte so that, e.g., int 1 and string "1" cannot render identically. const ( ctxTagString byte = 's' @@ -667,39 +666,35 @@ const ( ctxTagOther byte = 'o' ) -// hashContext computes a deterministic 64-bit FNV-1a hash of the pruned context map, used as -// the contextHash discriminator inside evaluationAggregationKey. +// canonicalContextKey builds the EXACT, comparable string key for the pruned context map, +// used as the contextKey field of evaluationAggregationKey. // -// The per-field encoding is CANONICAL — each field is a length-delimited key followed by a -// type-tag byte and a length-delimited value — so distinct contexts cannot ALIAS by -// construction (int 1 vs string "1" differ by tag; '=' / '\n' cannot fake a field boundary). -// Any digest collision is thus a genuine 64-bit hash collision, which is count-preserving (see -// the evaluationAggregationKey doc). -func hashContext(attrs map[string]any) uint64 { +// The encoding is CANONICAL — each field is a length-delimited key followed by a type-tag byte +// and a length-delimited value — so distinct contexts cannot ALIAS by construction (int 1 vs +// string "1" differ by tag; '=' / '\n' cannot fake a field boundary). Unlike the prior FNV-1a +// digest, the full encoding is emitted AS THE KEY, so Go's map compares it byte-for-byte: there +// is no hash and therefore no hash collision, so distinct contexts ALWAYS land in distinct +// full-tier buckets (oleksii #2). The returned string is stored once per full-tier bucket. +func canonicalContextKey(attrs map[string]any) string { if len(attrs) == 0 { - return 0 + return "" } - // Hash over a deterministic key ordering. Go map iteration is randomized, - // and FNV-1a is order-sensitive, so ranging the map directly would produce a - // different hash for identical contexts and fragment aggregation buckets. + // Encode over a deterministic key ordering. Go map iteration is randomized, and the + // concatenated encoding is order-sensitive, so ranging the map directly would produce a + // different key for identical contexts and fragment aggregation buckets. keys := make([]string, 0, len(attrs)) for k := range attrs { keys = append(keys, k) } sort.Strings(keys) - h := fnv.New64a() - // Reuse one scratch buffer across fields and write the bytes directly. FNV's Write - // neither retains the slice nor allocates, so hashing an N-field context costs ~1 - // allocation instead of the per-field reflection allocation of fmt.Fprintf — the - // dominant per-evaluation cost of this hook for large contexts. + // Build the encoding into a single buffer, then convert once to a string for the key. The + // per-field append uses the same canonical, allocation-light path as before. var buf []byte for _, k := range keys { - buf = buf[:0] buf = appendLengthDelimited(buf, []byte(k)) // length-delimited key buf = appendContextValue(buf, attrs[k]) // tag + length-delimited value - _, _ = h.Write(buf) } - return h.Sum64() + return string(buf) } // appendLengthDelimited writes a fixed-width 8-byte big-endian length prefix followed by the @@ -759,7 +754,7 @@ func appendContextValue(buf []byte, v any) []byte { // // The kept subset is DETERMINISTIC: keys are sorted BEFORE the oversized-string skip and the // 256-field cap are applied, so two logically-identical contexts always prune to the exact -// same subset (and therefore the same hashContext digest). Ranging the map directly (Go map +// same subset (and therefore the same canonicalContextKey). Ranging the map directly (Go map // iteration is randomized) would cut a different 256-field subset each call and fragment // otherwise-identical contexts into separate aggregation buckets. func pruneContext(raw map[string]any) map[string]any { diff --git a/openfeature/flagevaluation_scale_test.go b/openfeature/flagevaluation_scale_test.go index 9fb32f19384..d24dd1bdefb 100644 --- a/openfeature/flagevaluation_scale_test.go +++ b/openfeature/flagevaluation_scale_test.go @@ -78,7 +78,7 @@ func legitimateDegradedCardinality(flags []scaleFlagShape) int { // (so counts accumulate). It returns the total number of add() calls made. // // The context cardinality knob (numContexts) is what splits the FULL tier: the full key includes -// targetingKey + contextHash, so more distinct subjects => more full buckets => earlier full-tier +// targetingKey + contextKey, so more distinct subjects => more full buckets => earlier full-tier // saturation => earlier cascade into degraded (and potentially ultra). func driveScale(agg *flagEvaluationAggregator, flags []scaleFlagShape, numContexts, evalsPerCombo int) int64 { nowMs := time.Now().UnixMilli() diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 18ccbff15ea..6a91b4ea1b3 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -552,10 +552,10 @@ func TestAggregatorCapOverflow(t *testing.T) { // TestPruneContextDeterministic guards Finding #1 (nondeterministic context pruning). // A >256-field context must prune to an IDENTICAL kept subset (and therefore an identical -// hash) on every call, and two independently-built maps with the same logical entries must -// hash equal. On the pre-fix code (map-range cap BEFORE sort) the kept subset is random, so -// the hash varies across iterations and identical logical contexts fragment into separate -// buckets. +// canonical key) on every call, and two independently-built maps with the same logical entries +// must produce an equal key. On the pre-fix code (map-range cap BEFORE sort) the kept subset is +// random, so the key varies across iterations and identical logical contexts fragment into +// separate buckets. func TestPruneContextDeterministic(t *testing.T) { const fields = 400 // > maxContextFields (256) build := func() map[string]any { @@ -566,17 +566,17 @@ func TestPruneContextDeterministic(t *testing.T) { return m } - first := hashContext(pruneContext(build())) + first := canonicalContextKey(pruneContext(build())) for range 50 { - got := hashContext(pruneContext(build())) + got := canonicalContextKey(pruneContext(build())) if got != first { - t.Fatalf("pruneContext+hashContext nondeterministic over >256 fields: got %d, want %d", got, first) + t.Fatalf("pruneContext+canonicalContextKey nondeterministic over >256 fields: keys differ across iterations") } } - // Two independently-built maps with the SAME 400 logical entries must hash equal. - if a, b := hashContext(pruneContext(build())), hashContext(pruneContext(build())); a != b { - t.Errorf("identical logical contexts hashed differently: %d != %d", a, b) + // Two independently-built maps with the SAME 400 logical entries must produce an equal key. + if a, b := canonicalContextKey(pruneContext(build())), canonicalContextKey(pruneContext(build())); a != b { + t.Errorf("identical logical contexts produced different canonical keys") } } @@ -595,11 +595,11 @@ func TestPruneContextOversizedStringDeterministic(t *testing.T) { return m } - first := hashContext(pruneContext(build())) + first := canonicalContextKey(pruneContext(build())) for range 50 { - got := hashContext(pruneContext(build())) + got := canonicalContextKey(pruneContext(build())) if got != first { - t.Fatalf("oversized-string prune nondeterministic: got %d, want %d", got, first) + t.Fatalf("oversized-string prune nondeterministic: canonical keys differ across iterations") } } @@ -610,12 +610,13 @@ func TestPruneContextOversizedStringDeterministic(t *testing.T) { } } -// TestHashContextCanonicalEncoding guards Finding #2 (non-canonical context hash encoding). -// Distinct contexts must not alias: int 1 vs string "1" must differ, and '='/'\n'-bearing -// values/keys must not be able to fake a multi-field context. Drives a subset through -// aggregator.add and asserts SEPARATE full-tier buckets. -func TestHashContextCanonicalEncoding(t *testing.T) { - // Distinct contexts must hash differently — no aliasing across type or delimiter tricks. +// TestCanonicalContextKeyEncoding guards oleksii #2 (comparable canonical-context key replaces +// the lossy FNV-1a discriminator). Distinct contexts must produce DISTINCT keys — int 1 vs +// string "1" must differ, and '='/'\n'-bearing values/keys must not fake a multi-field context. +// Because the full canonical encoding IS the map key (no hash), distinct contexts ALWAYS land +// in separate full-tier buckets via add() with ZERO misattribution. +func TestCanonicalContextKeyEncoding(t *testing.T) { + // Distinct contexts must produce distinct keys — no aliasing across type or delimiter tricks. inequalityTests := []struct { name string mapA, mapB map[string]any @@ -643,23 +644,64 @@ func TestHashContextCanonicalEncoding(t *testing.T) { for _, tc := range inequalityTests { t.Run(tc.name, func(t *testing.T) { - if hashContext(tc.mapA) == hashContext(tc.mapB) { - t.Errorf("canonical encoding must distinguish %v from %v", tc.mapA, tc.mapB) + if canonicalContextKey(tc.mapA) == canonicalContextKey(tc.mapB) { + t.Errorf("canonical key must distinguish %v from %v", tc.mapA, tc.mapB) } }) } - t.Run("distinct contexts land in separate full-tier buckets via add()", func(t *testing.T) { + // Logically-identical contexts must produce the SAME key (so they aggregate into one bucket). + t.Run("identical contexts produce identical keys", func(t *testing.T) { + a := canonicalContextKey(map[string]any{"x": 1, "y": "two"}) + b := canonicalContextKey(map[string]any{"y": "two", "x": 1}) + if a != b { + t.Errorf("logically-identical contexts produced different canonical keys") + } + }) + + // Each distinct-context case must land in its OWN full-tier bucket via add() — the count is + // never misattributed to the other context (the defect the lossy FNV discriminator risked). + for _, tc := range inequalityTests { + t.Run("distinct buckets via add(): "+tc.name, func(t *testing.T) { + agg := setupTestAggregator(t) + nowMs := time.Now().UnixMilli() + d := evalDetails{flagKey: "f", variant: "on", reason: "targeting_match"} + agg.add(d, tc.mapA, nowMs) + agg.add(d, tc.mapB, nowMs) + + agg.mu.Lock() + defer agg.mu.Unlock() + if len(agg.full) != 2 { + t.Errorf("expected 2 full-tier buckets for distinct contexts %v vs %v, got %d", tc.mapA, tc.mapB, len(agg.full)) + } + // Zero misattribution: every bucket holds exactly the one count it received. + for k, e := range agg.full { + if e.count != 1 { + t.Errorf("bucket %+v has count %d; distinct contexts must not merge (misattribution)", k, e.count) + } + } + }) + } + + // A multi-field context with a key/value containing '\n' and '=' must still aggregate + // identically with itself (re-adding increments the SAME bucket, not a third). + t.Run("re-adding identical multi-field context increments same bucket", func(t *testing.T) { agg := setupTestAggregator(t) nowMs := time.Now().UnixMilli() d := evalDetails{flagKey: "f", variant: "on", reason: "targeting_match"} - agg.add(d, map[string]any{"x": 1}, nowMs) - agg.add(d, map[string]any{"x": "1"}, nowMs) + ctx := map[string]any{"a": "x\ny", "b": 7, "c=d": true} + agg.add(d, ctx, nowMs) + agg.add(d, map[string]any{"b": 7, "a": "x\ny", "c=d": true}, nowMs) // same logical context, different insertion order agg.mu.Lock() defer agg.mu.Unlock() - if len(agg.full) != 2 { - t.Errorf("expected 2 full-tier buckets for int 1 vs string \"1\", got %d", len(agg.full)) + if len(agg.full) != 1 { + t.Errorf("expected 1 full-tier bucket for identical context, got %d", len(agg.full)) + } + for _, e := range agg.full { + if e.count != 2 { + t.Errorf("expected count 2 for re-added identical context, got %d", e.count) + } } }) } From 4fad72fb900ee5dd4455391076c95e7acf6237e2 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 11:13:34 -0400 Subject: [PATCH 21/37] perf(openfeature): merge flatten+prune into one EVP worker-path traversal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address oleksii PR #4886 review #4: the worker path previously ran flattenContext (flatten.go) then pruneContext as two separate steps, each allocating its own map. Merge them into a single flattenAndPruneContext used by aggregate(): flatten nested objects, then apply the deterministic sort-before-cut 256-field / 256-char prune in one pass. Output is byte-for-byte identical to the prior two-step pipeline (same surviving keys, same 256/256 limits, same deterministic cut). When the flattened context already fits the limits — the common case — the flattened map is returned directly, eliding the second pruned-output map the old pipeline always allocated. flattenContext is left unchanged because exposure_hook.go still calls it; only the EVP aggregation path is merged. pruneContext is retained as the reference behavior for tests. Added TestFlattenAndPruneContextEquivalence asserting the merged pass equals flattenContext+pruneContext across nested, oversized, and >256-field inputs, with preserved determinism. --- openfeature/flagevaluation.go | 79 ++++++++++++++++++++++++++++-- openfeature/flagevaluation_test.go | 74 ++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 3 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index d412317bb14..f97d4aff3b1 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -490,17 +490,90 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int } } -// aggregate performs the deferred flatten/prune/hash and updates the aggregator. It runs +// aggregate performs the deferred flatten/prune/key and updates the aggregator. It runs // only on the writer's single worker goroutine. func (w *flagEvaluationWriter) aggregate(ev evalEvent) { var contextAttrs map[string]any if len(ev.attrs) > 0 { - flattened := flattenContext(ev.attrs) - contextAttrs = pruneContext(flattened) + contextAttrs = flattenAndPruneContext(ev.attrs) } w.aggregator.add(ev.d, contextAttrs, ev.nowMs) } +// flattenAndPruneContext is the worker-path procedure that produces the pruned context map for +// EVP aggregation in a SINGLE traversal of the flattened keyspace (oleksii #4). It merges the +// two former steps — flattenContext (flatten.go) + pruneContext — into one pass with the SAME +// pruned output: +// +// 1. Flatten nested objects into a single-level dot-notation map (reusing flattenRecursive, so +// the flatten semantics stay identical to the exposure path which still calls +// flattenContext directly — that caller is unchanged). +// 2. Apply the deterministic prune: sort the flattened keys, then keep the first +// maxContextFields that are not oversized strings (>maxFieldLength). +// +// Allocation win: when the flattened context already fits the limits (the common case — fewer +// than maxContextFields fields and no oversized string), the flattened map is returned DIRECTLY, +// so the separate pruned-output map that the old flatten→prune pipeline always allocated is +// elided. The pruned map is allocated only when trimming actually changes the result. Output is +// byte-for-byte identical to the previous flattenContext+pruneContext pipeline: same surviving +// keys, same 256/256 limits, same deterministic ordering of the cut. +func flattenAndPruneContext(attrs map[string]any) map[string]any { + if len(attrs) == 0 { + return nil + } + + flat := make(map[string]any, len(attrs)) + flattenRecursive("", attrs, flat) + if len(flat) == 0 { + return nil + } + + // Determine whether any pruning is actually required: an over-cap field count or any + // oversized string value. If neither, the flattened map already IS the pruned result — + // return it directly and skip allocating a second map. + needsPrune := len(flat) > maxContextFields + if !needsPrune { + for _, v := range flat { + if s, ok := v.(string); ok && len(s) > maxFieldLength { + needsPrune = true + break + } + } + } + if !needsPrune { + return flat + } + + // Deterministic prune: sort keys, then keep the first maxContextFields non-oversized values. + // Sorting BEFORE the oversized-string skip and the field cap makes the kept subset stable + // across calls (Go map iteration is randomized), so logically-identical contexts always + // prune to the same subset and the same canonicalContextKey. + keys := make([]string, 0, len(flat)) + for k := range flat { + keys = append(keys, k) + } + sort.Strings(keys) + + out := make(map[string]any, min(len(flat), maxContextFields)) + count := 0 + for _, k := range keys { + if count >= maxContextFields { + break + } + v := flat[k] + if s, ok := v.(string); ok && len(s) > maxFieldLength { + // Skip oversized string values (matches worker.ts pruneFields behavior). + continue + } + out[k] = v + count++ + } + if len(out) == 0 { + return nil + } + return out +} + // drainAndFlush processes any buffered events and performs a final flush. Called by the // worker when stopping so a final batch is not lost on shutdown. func (w *flagEvaluationWriter) drainAndFlush() { diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 6a91b4ea1b3..c2cdddfbbcd 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -8,6 +8,7 @@ package openfeature import ( "encoding/json" "fmt" + "reflect" "strings" "sync" "testing" @@ -16,6 +17,79 @@ import ( of "github.com/open-feature/go-sdk/openfeature" ) +// TestFlattenAndPruneContextEquivalence guards oleksii #4: the merged single-pass +// flattenAndPruneContext must produce a pruned result byte-for-byte identical to the prior +// two-step flattenContext + pruneContext pipeline across nested, oversized, and >256-field +// inputs (and the determinism + 256/256 limits are preserved). +func TestFlattenAndPruneContextEquivalence(t *testing.T) { + bigFields := func() map[string]any { + m := make(map[string]any, 400) + for i := range 400 { + m[fmt.Sprintf("key%04d", i)] = fmt.Sprintf("value%04d", i) + } + return m + } + + cases := []struct { + name string + input map[string]any + }{ + { + name: "nested objects flatten to dot notation", + input: map[string]any{"user": map[string]any{"id": "123", "email": "a@b.com"}, "country": "US"}, + }, + { + name: "deeply nested + arrays", + input: map[string]any{"a": map[string]any{"b": map[string]any{"c": 1}}, "tags": []string{"x", "y", "z"}}, + }, + { + name: "oversized string value is skipped", + input: map[string]any{"short": "ok", "long": strings.Repeat("x", maxFieldLength+10)}, + }, + { + name: "more than 256 fields truncated to 256", + input: bigFields(), + }, + { + name: "nested oversized string among many fields", + input: map[string]any{"u": map[string]any{"bio": strings.Repeat("y", maxFieldLength+1), "id": 7}}, + }, + { + name: "mixed scalar types retained", + input: map[string]any{"i": 42, "b": true, "f": 3.14, "s": "hi"}, + }, + { + name: "empty input", + input: map[string]any{}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Reference pipeline (the two former steps) vs the merged single-pass procedure. + want := pruneContext(flattenContext(tc.input)) + got := flattenAndPruneContext(tc.input) + + if !reflect.DeepEqual(got, want) { + t.Errorf("merged flatten+prune differs from flattenContext+pruneContext:\n got=%v\nwant=%v", got, want) + } + + // 256-field limit preserved. + if len(got) > maxContextFields { + t.Errorf("merged result has %d fields, exceeds maxContextFields %d", len(got), maxContextFields) + } + + // Determinism: repeated calls yield an identical canonical key. + first := canonicalContextKey(flattenAndPruneContext(tc.input)) + for range 25 { + if k := canonicalContextKey(flattenAndPruneContext(tc.input)); k != first { + t.Fatalf("merged flatten+prune nondeterministic: canonical keys differ across calls") + } + } + }) + } +} + // setupTestAggregator creates a flagEvaluationAggregator with small caps for testing. // Caps are deliberately small to trigger tier-cascade behavior in unit tests. func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { From 5422dfe54fde91c1c65d9d1785e8d59aa9d20271 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 12:05:00 -0400 Subject: [PATCH 22/37] feat(openfeature): record EVP eval-time from provider metadata, not hook-fire time (review #3) Capture the evaluation timestamp once at DatadogProvider.evaluate entry, thread it into evaluateFlag (reusing the time.Now() the evaluator already computes for allocation windows), and stamp it into FlagMetadata on every return path. The EVP flagevaluation hook reads it for first/last_evaluation bounds instead of calling time.Now() at hook-fire time (slightly later and slightly incorrect). Falls back to hook-fire time when absent (non-Datadog provider). flageval_metrics.go untouched. --- openfeature/evaluator.go | 10 ++-- openfeature/evaluator_test.go | 4 +- openfeature/exposure_hook.go | 5 ++ openfeature/flagevaluation.go | 19 +++++++- openfeature/flagevaluation_hook_test.go | 63 +++++++++++++++++++++++++ openfeature/provider.go | 17 ++++++- openfeature/provider_test.go | 37 +++++++++++++++ 7 files changed, 145 insertions(+), 10 deletions(-) diff --git a/openfeature/evaluator.go b/openfeature/evaluator.go index de581309b2d..b9c38150430 100644 --- a/openfeature/evaluator.go +++ b/openfeature/evaluator.go @@ -34,9 +34,12 @@ type evaluationResult struct { Metadata map[string]any } -// evaluateFlag evaluates a feature flag with the given context. +// evaluateFlag evaluates a feature flag with the given context. The caller supplies the +// evaluation time (now) so a single timestamp is shared between the allocation time-window +// checks here and the EVP eval-time metadata stamped by DatadogProvider.evaluate — avoiding a +// second time.Now() on the evaluation path. // It returns the variant value, reason, and any error that occurred. -func evaluateFlag(flag *flag, defaultValue any, context map[string]any) evaluationResult { +func evaluateFlag(flag *flag, defaultValue any, context map[string]any, now time.Time) evaluationResult { if flag == nil { return evaluationResult{Value: defaultValue, Reason: of.DefaultReason} } @@ -48,8 +51,7 @@ func evaluateFlag(flag *flag, defaultValue any, context map[string]any) evaluati } } - // Evaluate allocations in order - first match wins - now := time.Now() + // Evaluate allocations in order - first match wins (using the caller-supplied eval time) for _, allocation := range flag.Allocations { split, matched, err := evaluateAllocation(allocation, context, now) if err != nil { diff --git a/openfeature/evaluator_test.go b/openfeature/evaluator_test.go index d75b58540f9..0969c6130c8 100644 --- a/openfeature/evaluator_test.go +++ b/openfeature/evaluator_test.go @@ -703,7 +703,7 @@ func TestEvaluateFlag_VariantTypeMismatchReturnsParseError(t *testing.T) { }, } - result := evaluateFlag(flag, nil, map[string]any{"targetingKey": "user-123"}) + result := evaluateFlag(flag, nil, map[string]any{"targetingKey": "user-123"}, time.Now()) if result.Reason != of.ErrorReason { t.Errorf("expected ErrorReason, got %s", result.Reason) @@ -768,7 +768,7 @@ func TestEvaluateFlag_JSONFixtures(t *testing.T) { ctx["targetingKey"] = *tc.TargetingKey } - result := evaluateFlag(cfg.Flags[tc.Flag], tc.DefaultValue, ctx) + result := evaluateFlag(cfg.Flags[tc.Flag], tc.DefaultValue, ctx, time.Now()) if fmt.Sprintf("%v", result.Value) != fmt.Sprintf("%v", tc.Result.Value) { t.Errorf("value: got %v, want %v", result.Value, tc.Result.Value) diff --git a/openfeature/exposure_hook.go b/openfeature/exposure_hook.go index 301f72a7437..c1a47879212 100644 --- a/openfeature/exposure_hook.go +++ b/openfeature/exposure_hook.go @@ -19,6 +19,11 @@ const ( // Metadata keys for exposure tracking metadataAllocationKey = "dd.allocation.key" metadataDoLogKey = "dd.doLog" + // metadataEvalTimeKey carries the evaluation timestamp (UnixMilli, int64). It is stamped in + // DatadogProvider.evaluate at evaluation entry — the most-correct evaluation time — so the EVP + // flagevaluation hook records eval-time rather than the later, slightly-incorrect hook-fire + // time. Read back in extractEvalDetails (flagevaluation.go). + metadataEvalTimeKey = "dd.eval.timestamp_ms" ) // exposureHook implements the OpenFeature Hook interface to track feature flag exposures. diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index f97d4aff3b1..ed76244b81a 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -258,6 +258,10 @@ type evalDetails struct { targetingKey string errorMessage string runtimeDefault bool + // evalTimeMs is the evaluation timestamp (UnixMilli) captured by the provider at eval entry + // and passed through flag metadata. 0 when absent (e.g. a non-Datadog provider), in which case + // record() falls back to the hook-fire time. + evalTimeMs int64 } // newFlagEvaluationWriter creates a new flag evaluation writer. @@ -478,10 +482,18 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int w.dropped.Add(1) return } + d := extractEvalDetails(hookContext, details) + // Use the evaluation time captured by the provider (most-correct; see metadataEvalTimeKey). + // Fall back to the hook-fire time only when absent (e.g. a non-Datadog provider that did not + // stamp it), so the first/last_evaluation bounds are always populated. + nowMs := d.evalTimeMs + if nowMs == 0 { + nowMs = time.Now().UnixMilli() + } ev := evalEvent{ - d: extractEvalDetails(hookContext, details), + d: d, attrs: hookContext.EvaluationContext().Attributes(), // SDK returns an owned copy - nowMs: time.Now().UnixMilli(), + nowMs: nowMs, } select { case w.events <- ev: @@ -876,6 +888,8 @@ func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluati if errMsg == "" && details.ErrorCode != "" { errMsg = string(details.ErrorCode) } + // Evaluation time, stamped by DatadogProvider.evaluate at eval entry. 0 when absent. + evalTimeMs, _ := details.FlagMetadata[metadataEvalTimeKey].(int64) return evalDetails{ flagKey: hookContext.FlagKey(), variant: details.Variant, @@ -884,5 +898,6 @@ func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluati targetingKey: hookContext.EvaluationContext().TargetingKey(), errorMessage: errMsg, runtimeDefault: isRuntimeDefault(details), + evalTimeMs: evalTimeMs, } } diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index e170aadcdac..ac154975eb3 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -136,6 +136,69 @@ func TestIsRuntimeDefault(t *testing.T) { } } +// TestExtractEvalDetailsReadsEvalTime verifies the provider-stamped evaluation time +// (metadataEvalTimeKey) is read into evalDetails.evalTimeMs, and is 0 when absent. +func TestExtractEvalDetailsReadsEvalTime(t *testing.T) { + const evalTime int64 = 1_717_171_717_123 + tests := []struct { + name string + md of.FlagMetadata + want int64 + }{ + {"eval-time present", of.FlagMetadata{metadataEvalTimeKey: evalTime}, evalTime}, + {"eval-time present alongside allocation key", of.FlagMetadata{metadataAllocationKey: "alloc-1", metadataEvalTimeKey: evalTime}, evalTime}, + {"eval-time absent", of.FlagMetadata{metadataAllocationKey: "alloc-1"}, 0}, + {"nil metadata", nil, 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + hookCtx := makeHookContext("flag-x", "user-1", nil) + details := makeEvalDetails("on", of.TargetingMatchReason, "", tc.md) + if got := extractEvalDetails(hookCtx, details).evalTimeMs; got != tc.want { + t.Errorf("evalTimeMs = %d, want %d", got, tc.want) + } + }) + } +} + +// TestRecordUsesEvalTimeFromMetadata verifies record() stamps the aggregated entry's +// first/last evaluation from the provider-supplied eval-time (metadataEvalTimeKey), and +// falls back to the hook-fire time only when that metadata is absent. Fails on the prior +// behavior, which always used time.Now() in record(). +func TestRecordUsesEvalTimeFromMetadata(t *testing.T) { + t.Run("uses provider eval-time", func(t *testing.T) { + const evalTime int64 = 1_700_000_000_000 // a fixed time in the past, distinct from now + w := setupTestWriter(t) + w.record(makeHookContext("flag-y", "user-2", nil), + makeEvalDetails("on", of.TargetingMatchReason, "", of.FlagMetadata{metadataEvalTimeKey: evalTime})) + w.aggregate(<-w.events) + + if len(w.aggregator.full) != 1 { + t.Fatalf("expected 1 full-tier entry, got %d", len(w.aggregator.full)) + } + for _, e := range w.aggregator.full { + if e.firstEvaluation != evalTime || e.lastEvaluation != evalTime { + t.Errorf("entry first/last = %d/%d, want %d (provider eval-time)", e.firstEvaluation, e.lastEvaluation, evalTime) + } + } + }) + + t.Run("falls back to hook-fire time when absent", func(t *testing.T) { + before := time.Now().UnixMilli() + w := setupTestWriter(t) + w.record(makeHookContext("flag-z", "user-3", nil), + makeEvalDetails("on", of.TargetingMatchReason, "")) // no eval-time metadata + w.aggregate(<-w.events) + after := time.Now().UnixMilli() + + for _, e := range w.aggregator.full { + if e.firstEvaluation < before || e.firstEvaluation > after { + t.Errorf("fallback first=%d not within hook-fire window [%d,%d]", e.firstEvaluation, before, after) + } + } + }) +} + // TestFlagEvaluationHookFinally verifies that the Finally hook records an entry for // success, error-reason, and provider-not-ready paths. // diff --git a/openfeature/provider.go b/openfeature/provider.go index de606652bee..3f79537cf25 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -469,8 +469,21 @@ func (p *DatadogProvider) evaluate( defaultValue any, flatCtx openfeature.FlattenedContext, ) (res evaluationResult) { + // Capture the evaluation time once, at evaluation entry — the most-correct "eval time". It is + // reused for the allocation time-window checks (passed into evaluateFlag) and stamped into the + // result metadata below, so the EVP flagevaluation hook records eval-time instead of the later + // hook-fire time, with no extra time.Now() on the eval path. + evalNow := time.Now() log.Debug("openfeature: evaluating flag %q", flagKey) defer func() { + // Stamp eval-time into metadata on EVERY return path (matched, default, disabled, error, + // ctx-cancelled, no-config, not-found). On the matched path the metadata map already exists + // (allocation key + doLog), so this only adds a key; on other paths it allocates a 1-entry + // map. Read back by extractEvalDetails for the EVP first/last_evaluation bounds. + if res.Metadata == nil { + res.Metadata = make(map[string]any, 1) + } + res.Metadata[metadataEvalTimeKey] = evalNow.UnixMilli() log.Debug("openfeature: evaluated flag %q: value=%v, reason=%s, error=%v", flagKey, res.Value, res.Reason, res.Error) }() @@ -506,8 +519,8 @@ func (p *DatadogProvider) evaluate( } } - // Evaluate the flag (pass context for potential future use in evaluateFlag) - return evaluateFlag(flag, defaultValue, flatCtx) + // Evaluate the flag, sharing the eval-time captured at entry. + return evaluateFlag(flag, defaultValue, flatCtx, evalNow) } // toResolutionError converts a Go error to an OpenFeature ResolutionError. diff --git a/openfeature/provider_test.go b/openfeature/provider_test.go index 1e5f088ce1d..52f537988d2 100644 --- a/openfeature/provider_test.go +++ b/openfeature/provider_test.go @@ -262,6 +262,43 @@ func TestNewDatadogProvider(t *testing.T) { } } +// TestEvaluateStampsEvalTimeMetadata verifies the provider stamps the evaluation time into +// FlagMetadata (metadataEvalTimeKey) on EVERY path — matched, default, disabled, not-found — so +// the EVP flagevaluation hook records eval-time rather than the later hook-fire time. +func TestEvaluateStampsEvalTimeMetadata(t *testing.T) { + provider := newDatadogProvider(ProviderConfig{}) + provider.updateConfiguration(createTestConfig()) + ctx := context.Background() + + cases := []struct { + name string + flagKey string + flatCtx openfeature.FlattenedContext + }{ + {"matched allocation", "bool-flag", openfeature.FlattenedContext{"targetingKey": "user-123", "country": "US"}}, + {"default (no match)", "bool-flag", openfeature.FlattenedContext{"targetingKey": "user-123", "country": "CA"}}, + {"disabled flag", "disabled-flag", openfeature.FlattenedContext{"targetingKey": "user-123"}}, + {"flag not found", "nonexistent-flag", openfeature.FlattenedContext{"targetingKey": "user-123"}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + before := time.Now().UnixMilli() + result := provider.BooleanEvaluation(ctx, tc.flagKey, false, tc.flatCtx) + after := time.Now().UnixMilli() + + ts, ok := result.FlagMetadata[metadataEvalTimeKey].(int64) + if !ok { + t.Fatalf("FlagMetadata[%q] missing or not int64 on %q path; metadata=%v", + metadataEvalTimeKey, tc.name, result.FlagMetadata) + } + if ts < before || ts > after { + t.Errorf("eval-time %d not within evaluation window [%d,%d]", ts, before, after) + } + }) + } +} + func TestBooleanEvaluation(t *testing.T) { provider := newDatadogProvider(ProviderConfig{}) config := createTestConfig() From e2441ed36bcf19763e7675a5ac0aa2ac6f699f37 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 02:53:49 -0400 Subject: [PATCH 23/37] fix(openfeature): align EVP flagevaluation keys with worker schema --- go.mod | 1 + go.sum | 2 + openfeature/flagevaluation.go | 159 +++++++------ openfeature/flagevaluation_hook.go | 8 +- openfeature/flagevaluation_hook_test.go | 5 + openfeature/flagevaluation_scale_test.go | 82 +++---- openfeature/flagevaluation_test.go | 208 +++++++++++++++--- openfeature/provider_bench_test.go | 4 +- .../batchedflagevaluations.json | 184 ++++++++++++++++ 9 files changed, 493 insertions(+), 160 deletions(-) create mode 100644 openfeature/testdata/flageval-worker/batchedflagevaluations.json diff --git a/go.mod b/go.mod index aa70db14c5c..0328145f64e 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/stretchr/objx v0.5.3 // indirect diff --git a/go.sum b/go.sum index 02f5751071a..caa8e8bba26 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVO github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3/go.mod h1:vl5+MqJ1nBINuSsUI2mGgH79UweUT/B5Fy8857PqyyI= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14= github.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index ed76244b81a..244210de1aa 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -17,7 +17,6 @@ import ( "os" "sort" "strconv" - "strings" "sync" "sync/atomic" "time" @@ -47,15 +46,13 @@ const ( // Aggregation caps. // - // SPIKE cq7 (2-tier resize): the cascade is now full → degraded → drop(counted). With no - // ultra-degraded backstop, degradedCap must hold the FULL legitimate degraded cardinality at - // the team's >=2,500-flag scale target, or legitimate counts would be dropped. The measured - // legitimate degraded cardinality at 2,500 flags (Σ variants×allocations×reasons over a - // realistic flag mix) is 21,662 distinct buckets (see flagevaluation_scale_test.go); so - // degradedCap is raised to 32,768 (~1.5x headroom). globalCap (full-tier) is raised to + // The cascade is full → degraded → drop(counted). With no ultra-degraded backstop, + // degradedCap must hold the legitimate degraded cardinality at the team's >=2,500-flag + // scale target, or legitimate counts would be dropped. Degraded cardinality is based only + // on schema-visible dimensions retained by the degraded payload; OpenFeature reason is not + // an EVP field and is not part of bucket identity. globalCap (full-tier) is raised to // 131,072 so a realistic 2,500-flag × multi-context workload keeps full-fidelity buckets - // before degrading rather than dropping them — overflow from full still cascades to degraded, - // which is now sized to hold it. + // before degrading rather than dropping them. defaultEvalGlobalCap = 131_072 // bounds full-tier buckets only; degraded is bounded separately defaultEvalPerFlagCap = 10_000 // bounds full-fidelity buckets per flag defaultEvalDegradedCap = 32_768 // bounds degraded map; overflow is dropped(counted), no ultra tier @@ -70,11 +67,11 @@ const ( // EXACT, comparable string compared byte-for-byte by Go map equality, so distinct keys can // NEVER alias into the same bucket: // -// - The enumerable dimensions (flagKey, variant, allocationKey, reason, targetingKey) are -// the raw evaluation strings. +// - The enumerable dimensions are schema-visible fields only. OpenFeature reason is not +// accepted by the flageval-worker schema and must never be hidden cardinality. // - contextKey is the EXACT canonical type-tagged, length-delimited encoding of the pruned // context (see canonicalContextKey). Because it is the full encoding — not a lossy 64-bit -// digest — two distinct contexts ALWAYS produce distinct contextKey strings (oleksii #2): +// digest — two distinct contexts ALWAYS produce distinct contextKey strings: // int 1 vs string "1" differ by type tag, and '='/'\n'-bearing values cannot fake a field // boundary thanks to the length prefixes. There is no hash, so there is no hash collision, // so a count can never be misattributed to the wrong context. Go's map hashes and compares @@ -84,24 +81,26 @@ const ( // bounded by the number of full-tier buckets (globalCap) and the pruned context size // (256 fields × 256 chars), and measured in BenchmarkFlagEvaluationOTelPlusEVPParallel. type evaluationAggregationKey struct { - flagKey string - variant string - allocationKey string - reason string - targetingKey string - contextKey string // exact canonical encoding of the pruned context; comparable, not a digest + flagKey string + variant string + allocationKey string + runtimeDefault bool + errorMessage string + targetingKey string + contextKey string // exact canonical encoding of the pruned context; comparable, not a digest } // evaluationDegradedKey is the key for the degraded aggregation map — the terminal aggregation // tier in the 2-tier design (full → degraded → drop). Drops targeting key, context, and -// targeting rule key relative to the full key. This is EXACTLY the OTel feature_flag.evaluations -// cardinality. When a NEW degraded bucket would exceed degradedCap, the count is dropped and -// counted (aggregator.dropped) rather than cascading to a further-degraded tier. +// targeting rule key relative to the full key. It keeps only schema-visible fields emitted by +// the degraded payload. When a NEW degraded bucket would exceed degradedCap, the count is +// dropped and counted (aggregator.dropped) rather than cascading to a further-degraded tier. type evaluationDegradedKey struct { - flagKey string - variant string - allocationKey string - reason string + flagKey string + variant string + allocationKey string + runtimeDefault bool + errorMessage string } // evaluationEntry holds per-window counts and time bounds for one aggregation bucket. @@ -229,23 +228,22 @@ type flagEvaluationWriter struct { stopped atomic.Bool // single idempotency gate for stop(); also read lock-free by record() jsonConfig jsoniter.API - // Asynchronous hand-off: the Finally hook enqueues a cheap snapshot here; a single - // background worker (started in start()) drains it and performs flatten/prune/hash/ - // aggregate off the evaluation hot path. events is bounded; on overflow the hook drops + // Asynchronous hand-off: the Finally hook enqueues a bounded snapshot here; a single + // background worker (started in start()) drains it and performs aggregate/flush off the + // evaluation hot path. events is bounded; on overflow the hook drops // the event and bumps dropped — best-effort telemetry that never blocks the request. events chan evalEvent dropped atomic.Int64 workerDone chan struct{} + enqueueMu sync.RWMutex } -// evalEvent is the minimal snapshot the Finally hook hands to the worker. attrs is the -// owned copy returned by EvaluationContext().Attributes(), so it is safe to read off the -// hot path for scalar values (a nested mutable attribute the caller mutates after the call -// returns is a documented best-effort edge). +// evalEvent is the bounded snapshot the Finally hook hands to the worker. contextAttrs is already +// flattened and pruned, so the async queue never buffers the caller's raw evaluation context. type evalEvent struct { - d evalDetails - attrs map[string]any - nowMs int64 + d evalDetails + contextAttrs map[string]any + nowMs int64 } // evalDetails holds extracted flag evaluation fields for EVP aggregation. @@ -253,7 +251,6 @@ type evalEvent struct { type evalDetails struct { flagKey string variant string - reason string allocationKey string targetingKey string errorMessage string @@ -324,8 +321,7 @@ func (w *flagEvaluationWriter) start() { close(w.workerDone) }() - // Single owner of flatten/prune/hash/aggregate/flush. The hot path only enqueues; - // all that cost lives here, off the evaluation path. + // Single owner of aggregate/flush. The hook enqueues only bounded snapshots. for { select { case ev := <-w.events: @@ -342,16 +338,18 @@ func (w *flagEvaluationWriter) start() { // stop stops the flush ticker and marks the writer as stopped. func (w *flagEvaluationWriter) stop() { - // Single idempotency gate: the atomic Swap is the ONLY guard for both "mark stopped" and - // the downstream close(stopChan). A second/concurrent stop() sees Swap return true and - // returns early, so stopChan is closed exactly once (no double-close panic). record() reads - // this flag lock-free to no-op post-stop enqueues. + w.enqueueMu.Lock() + // Single idempotency gate: the atomic Swap is the guard for both "mark stopped" and the + // downstream close(stopChan). enqueueMu prevents a record() call that observed stopped=false + // from sending into events after the worker has drained and exited. if w.stopped.Swap(true) { + w.enqueueMu.Unlock() return } // Signal the worker to drain the queue and do a final flush. close(w.stopChan) + w.enqueueMu.Unlock() if w.ticker != nil { // Worker was started: wait for its final flush before returning, then stop the // ticker. (ticker is set only in start(), so it gates "was the worker started".) @@ -404,8 +402,8 @@ func (w *flagEvaluationWriter) flush() { nowMs := time.Now().UnixMilli() var events []flagEvaluationEvent - // Full tier: required fields + variant + allocation + targeting_key + context + error; - // runtime_default decorates this tier. + // Full tier: required fields + variant + allocation + targeting_key + context + error. + // runtime_default_used decorates this tier when the caller default was returned. for key, e := range full { ev := baseFlagEvaluationEvent(key.flagKey, e, nowMs) ev.RuntimeDefault = e.runtimeDefault @@ -425,8 +423,8 @@ func (w *flagEvaluationWriter) flush() { events = append(events, ev) } - // Degraded tier: required fields + variant + allocation; no targeting_key, no context. - // runtime_default decorates this tier. + // Degraded tier: required fields + variant + allocation + error; no targeting_key, no context. + // runtime_default_used decorates this tier when the caller default was returned. for key, e := range degraded { ev := baseFlagEvaluationEvent(key.flagKey, e, nowMs) ev.RuntimeDefault = e.runtimeDefault @@ -436,6 +434,9 @@ func (w *flagEvaluationWriter) flush() { if key.allocationKey != "" { ev.Allocation = &flagEvalAllocation{Key: key.allocationKey} } + if e.errorMessage != "" { + ev.Error = &flagEvalError{Message: e.errorMessage} + } events = append(events, ev) } @@ -470,11 +471,14 @@ func baseFlagEvaluationEvent(flagKey string, e *evaluationEntry, nowMs int64) fl } // record runs on the evaluation hot path (the Finally hook). It does only cheap scalar -// extraction plus the SDK's shallow context copy, then a non-blocking enqueue — no -// flatten/prune/hash/aggregation happens here; the background worker does that. If the -// queue is full the event is dropped and counted (best-effort), never blocking the -// evaluation. Called from the Finally hook after every evaluation. +// extraction plus a bounded context snapshot, then a non-blocking enqueue — no aggregation +// happens here; the background worker does that. If the queue is full the event is dropped +// and counted (best-effort), never blocking the evaluation. Called from the Finally hook after +// every evaluation. func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.InterfaceEvaluationDetails) { + w.enqueueMu.RLock() + defer w.enqueueMu.RUnlock() + // Post-stop no-op: after stop() the worker no longer drains w.events, so enqueuing would // silently lose the event. Check the atomic gate lock-free (reading under the aggregator // lock would add hot-path contention) and count the event as dropped so it stays observable. @@ -491,9 +495,9 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int nowMs = time.Now().UnixMilli() } ev := evalEvent{ - d: d, - attrs: hookContext.EvaluationContext().Attributes(), // SDK returns an owned copy - nowMs: nowMs, + d: d, + contextAttrs: flattenAndPruneContext(hookContext.EvaluationContext().Attributes()), + nowMs: nowMs, } select { case w.events <- ev: @@ -502,18 +506,13 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int } } -// aggregate performs the deferred flatten/prune/key and updates the aggregator. It runs -// only on the writer's single worker goroutine. +// aggregate updates the aggregator. It runs only on the writer's single worker goroutine. func (w *flagEvaluationWriter) aggregate(ev evalEvent) { - var contextAttrs map[string]any - if len(ev.attrs) > 0 { - contextAttrs = flattenAndPruneContext(ev.attrs) - } - w.aggregator.add(ev.d, contextAttrs, ev.nowMs) + w.aggregator.add(ev.d, ev.contextAttrs, ev.nowMs) } -// flattenAndPruneContext is the worker-path procedure that produces the pruned context map for -// EVP aggregation in a SINGLE traversal of the flattened keyspace (oleksii #4). It merges the +// flattenAndPruneContext produces the pruned context map for EVP aggregation in a single +// traversal of the flattened keyspace. It merges the // two former steps — flattenContext (flatten.go) + pruneContext — into one pass with the SAME // pruned output: // @@ -649,15 +648,16 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an a.mu.Lock() defer a.mu.Unlock() - // Build the full key: exact, comparable strings for every dimension including the canonical - // context encoding. No hash, so distinct contexts get distinct buckets (oleksii #2). + // Build the full key from schema-visible dimensions including the canonical context encoding. + // No hash, so distinct contexts get distinct buckets. fullKey := evaluationAggregationKey{ - flagKey: d.flagKey, - variant: d.variant, - allocationKey: d.allocationKey, - reason: d.reason, - targetingKey: d.targetingKey, - contextKey: canonicalContextKey(contextAttrs), + flagKey: d.flagKey, + variant: d.variant, + allocationKey: d.allocationKey, + runtimeDefault: d.runtimeDefault, + errorMessage: d.errorMessage, + targetingKey: d.targetingKey, + contextKey: canonicalContextKey(contextAttrs), } // Fast path: this exact full-tier bucket already exists → increment its count. Because @@ -710,14 +710,15 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an // DROPPED and counted (droppedDegradedOverflow) rather than cascading to a further-degraded // tier. degradedCap is sized (defaultEvalDegradedCap) to hold the legitimate degraded // cardinality at the >=2,500-flag scale target, so this drop only fires under cardinality far -// beyond that target (e.g. an unbounded dynamic/abusive flag key — reviewer concern #8) and the +// beyond that target (e.g. an unbounded dynamic/abusive flag key) and the // dropped counter makes such overflow observable. func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { degKey := evaluationDegradedKey{ - flagKey: d.flagKey, - variant: d.variant, - allocationKey: d.allocationKey, - reason: d.reason, + flagKey: d.flagKey, + variant: d.variant, + allocationKey: d.allocationKey, + runtimeDefault: d.runtimeDefault, + errorMessage: d.errorMessage, } if e, ok := a.degraded[degKey]; ok { @@ -735,6 +736,7 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { e := newEvaluationEntry(nowMs) e.runtimeDefault = d.runtimeDefault + e.errorMessage = d.errorMessage a.degraded[degKey] = e } @@ -759,7 +761,7 @@ const ( // string "1" differ by tag; '=' / '\n' cannot fake a field boundary). Unlike the prior FNV-1a // digest, the full encoding is emitted AS THE KEY, so Go's map compares it byte-for-byte: there // is no hash and therefore no hash collision, so distinct contexts ALWAYS land in distinct -// full-tier buckets (oleksii #2). The returned string is stored once per full-tier bucket. +// full-tier buckets. The returned string is stored once per full-tier bucket. func canonicalContextKey(attrs map[string]any) string { if len(attrs) == 0 { return "" @@ -878,10 +880,6 @@ func pruneContext(raw map[string]any) map[string]any { // flageval_metrics.go (that file is left untouched to preserve the OTel path). func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluationDetails) evalDetails { allocationKey, _ := details.FlagMetadata[metadataAllocationKey].(string) - reason := strings.ToLower(string(details.Reason)) - if reason == "" { - reason = "unknown" - } // Prefer OpenFeature's human-readable ErrorMessage; fall back to the ErrorCode string only // when ErrorMessage is empty (some providers populate just the code). errMsg := details.ErrorMessage @@ -893,7 +891,6 @@ func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluati return evalDetails{ flagKey: hookContext.FlagKey(), variant: details.Variant, - reason: reason, allocationKey: allocationKey, targetingKey: hookContext.EvaluationContext().TargetingKey(), errorMessage: errMsg, diff --git a/openfeature/flagevaluation_hook.go b/openfeature/flagevaluation_hook.go index f43838bd7d1..de1054cd924 100644 --- a/openfeature/flagevaluation_hook.go +++ b/openfeature/flagevaluation_hook.go @@ -45,9 +45,9 @@ func (h *flagEvaluationHook) Finally( } // isRuntimeDefault returns true when the caller's supplied default value was returned. -// Signal: absent variant key. Our evaluator sets a variant ONLY on a matched allocation -// (reason TARGETING_MATCH/SPLIT/STATIC); every DEFAULT/DISABLED/ERROR path leaves the variant -// empty (see evaluator.go). A present variant therefore means a real assignment, not a default. +// The primary signal is an absent variant key. Type mismatches are the exception: the provider +// may have produced a real variant, but the OpenFeature SDK returns the caller default after +// conversion fails. func isRuntimeDefault(details of.InterfaceEvaluationDetails) bool { - return details.Variant == "" + return details.Variant == "" || details.ErrorCode == of.TypeMismatchCode } diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index ac154975eb3..f9dfe16e887 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -124,6 +124,11 @@ func TestIsRuntimeDefault(t *testing.T) { details: makeEvalDetails("on", of.DefaultReason, ""), want: false, }, + { + name: "variant present with TYPE_MISMATCH is runtime default", + details: makeEvalDetails("on", of.ErrorReason, of.TypeMismatchCode), + want: true, + }, } for _, tc := range tests { diff --git a/openfeature/flagevaluation_scale_test.go b/openfeature/flagevaluation_scale_test.go index d24dd1bdefb..cc162d7aaf2 100644 --- a/openfeature/flagevaluation_scale_test.go +++ b/openfeature/flagevaluation_scale_test.go @@ -11,36 +11,26 @@ import ( "time" ) -// SPIKE (cq7): decisive measurement for the "remove the ultra-degraded tier" decision. -// // These tests drive flagEvaluationAggregator.add directly (no client, no hooks, no async // worker) to deterministically simulate the team's >=2,500-flag scale target with realistic -// flag structure, then INSPECT the three aggregation maps. The goal is to answer: +// flag structure, then inspect the two aggregation maps. The goal is to answer: // -// 1. At 2,500 flags, under what conditions (if any) does the ultra-degraded tier actually -// trigger? Does it require saturating BOTH full(65k) AND degraded(10k)? -// 2. Does 2,500 flags' worth of LEGITIMATE degraded buckets fit under degradedCap=10,000, or -// does it overflow (cascading to ultra today, DROPPING under a 2-tier design)? -// 3. What degradedCap/globalCap values make a 2-tier design never drop legitimate counts at +// 1. Does 2,500 flags' worth of legitimate schema-visible degraded buckets fit under degradedCap, +// or does it overflow and drop under a 2-tier design? +// 2. What degradedCap/globalCap values make a 2-tier design never drop legitimate counts at // 2,500 flags? // // They are not assertions about correct behavior so much as a measurement harness; each test // logs its numbers via t.Logf and is intended to be read with `go test -run TestScale -v`. // scaleFlagShape describes the realistic per-flag structure used to size the degraded tier. -// Degraded key = (flagKey, variant, allocationKey, reason), so the LEGITIMATE degraded -// cardinality of a flag is variants × allocations × reasons-actually-observed. +// Degraded key includes schema-visible retained fields only. OpenFeature reason is not accepted +// by the worker schema and is not part of degraded cardinality. type scaleFlagShape struct { variants int // distinct variant keys this flag can return allocations int // distinct allocation keys this flag can return - reasons []string } -// realisticReasons is the natural reason set a healthy server-side flag emits. A flag that -// matches targeting emits TARGETING_MATCH/SPLIT for assigned subjects and DEFAULT for the -// rest, so 2 reasons/flag is the realistic steady state (not all 8 canonical reasons). -var realisticReasons = []string{"targeting_match", "default"} - // makeScaleFlags builds n flags with a realistic spread of variants/allocations. The spread is // deterministic (driven by index modulo) so the degraded cardinality is exactly reproducible: // - ~1/3 of flags: 2 variants, 1 allocation (simple on/off) @@ -51,29 +41,29 @@ func makeScaleFlags(n int) []scaleFlagShape { for i := range n { switch i % 3 { case 0: - flags[i] = scaleFlagShape{variants: 2, allocations: 1, reasons: realisticReasons} + flags[i] = scaleFlagShape{variants: 2, allocations: 1} case 1: - flags[i] = scaleFlagShape{variants: 3, allocations: 1, reasons: realisticReasons} + flags[i] = scaleFlagShape{variants: 3, allocations: 1} default: - flags[i] = scaleFlagShape{variants: 4, allocations: 2, reasons: realisticReasons} + flags[i] = scaleFlagShape{variants: 4, allocations: 2} } } return flags } // legitimateDegradedCardinality returns the number of DISTINCT degraded buckets the given flag -// shapes would produce if every (variant × allocation × reason) combination were observed. +// shapes would produce if every (variant × allocation) combination were observed. // This is the count a 2-tier design must be able to hold without dropping. func legitimateDegradedCardinality(flags []scaleFlagShape) int { total := 0 for _, f := range flags { - total += f.variants * f.allocations * len(f.reasons) + total += f.variants * f.allocations } return total } // driveScale records evaluations into agg for the given flag shapes, distributing each flag's -// evaluations across its variant/allocation/reason combinations and across numContexts distinct +// evaluations across its variant/allocation combinations and across numContexts distinct // evaluation contexts (subjects). evalsPerCombo controls how many subjects hit each combination // (so counts accumulate). It returns the total number of add() calls made. // @@ -90,25 +80,22 @@ func driveScale(agg *flagEvaluationAggregator, flags []scaleFlagShape, numContex variant := fmt.Sprintf("v%d", v) for a := range f.allocations { alloc := fmt.Sprintf("alloc-%d", a) - for _, reason := range f.reasons { - for range evalsPerCombo { - // Spread subjects across numContexts distinct targeting keys + contexts. - subj := ctxCounter % numContexts - ctxCounter++ - d := evalDetails{ - flagKey: flagKey, - variant: variant, - allocationKey: alloc, - reason: reason, - targetingKey: fmt.Sprintf("user-%d", subj), - } - attrs := map[string]any{ - "country": fmt.Sprintf("c%d", subj%50), - "plan": fmt.Sprintf("p%d", subj%5), - } - agg.add(d, attrs, nowMs) - calls++ + for range evalsPerCombo { + // Spread subjects across numContexts distinct targeting keys + contexts. + subj := ctxCounter % numContexts + ctxCounter++ + d := evalDetails{ + flagKey: flagKey, + variant: variant, + allocationKey: alloc, + targetingKey: fmt.Sprintf("user-%d", subj), + } + attrs := map[string]any{ + "country": fmt.Sprintf("c%d", subj%50), + "plan": fmt.Sprintf("p%d", subj%5), } + agg.add(d, attrs, nowMs) + calls++ } } } @@ -166,10 +153,10 @@ func TestScaleDegradedCardinality2500Flags(t *testing.T) { } } t.Logf("2,500-flag realistic shape:") - t.Logf(" %d flags @ 2v×1a×2r = %d degraded buckets", s0, s0*2*1*2) - t.Logf(" %d flags @ 3v×1a×2r = %d degraded buckets", s1, s1*3*1*2) - t.Logf(" %d flags @ 4v×2a×2r = %d degraded buckets", s2, s2*4*2*2) - t.Logf("LEGITIMATE degraded cardinality (Σ variants×allocations×reasons) = %d", deg) + t.Logf(" %d flags @ 2v×1a = %d degraded buckets", s0, s0*2*1) + t.Logf(" %d flags @ 3v×1a = %d degraded buckets", s1, s1*3*1) + t.Logf(" %d flags @ 4v×2a = %d degraded buckets", s2, s2*4*2) + t.Logf("LEGITIMATE degraded cardinality (Σ variants×allocations) = %d", deg) t.Logf("production degradedCap = %d", defaultEvalDegradedCap) t.Logf("production globalCap = %d", defaultEvalGlobalCap) @@ -304,7 +291,7 @@ func TestScaleFullSaturationCascade(t *testing.T) { defaultEvalPerFlagCap, defaultEvalDegradedCap, ) - // legitimate degraded cardinality is ~21,662 combos; with 8 distinct subjects per combo the + // legitimate degraded cardinality is the schema-visible combo count; with enough distinct subjects per combo the // full tier sees ~173k distinct full keys, over globalCap, forcing overflow into degraded. calls := driveScale(agg, flags, 1_000_000, 8) tc := snapshot(agg) @@ -339,8 +326,8 @@ func TestScaleHotFlagPerFlagCap(t *testing.T) { nowMs := time.Now().UnixMilli() // One hot flag, many distinct (variant, subject) combos so it blows past perFlagCap and then - // keeps generating distinct degraded keys. Degraded key = (flag,variant,alloc,reason); to fill - // the resized degradedCap we need that many distinct (variant,alloc,reason) for this one flag. + // keeps generating distinct degraded keys. To fill the resized degradedCap we need that many + // distinct schema-visible variant/allocation combinations for this one flag. const distinctVariants = 50_000 var calls int64 for v := range distinctVariants { @@ -348,7 +335,6 @@ func TestScaleHotFlagPerFlagCap(t *testing.T) { flagKey: "hot-flag", variant: fmt.Sprintf("v%d", v), allocationKey: "alloc-0", - reason: "targeting_match", targetingKey: fmt.Sprintf("user-%d", v), } agg.add(d, map[string]any{"k": v}, nowMs) diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index c2cdddfbbcd..0169dbc2f04 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -8,16 +8,47 @@ package openfeature import ( "encoding/json" "fmt" + "os" "reflect" "strings" "sync" "testing" "time" + "github.com/santhosh-tekuri/jsonschema/v5" + of "github.com/open-feature/go-sdk/openfeature" ) -// TestFlattenAndPruneContextEquivalence guards oleksii #4: the merged single-pass +func validateBatchedFlagEvaluationsSchema(t *testing.T, payload any) error { + t.Helper() + + schemaBytes, err := os.ReadFile("testdata/flageval-worker/batchedflagevaluations.json") + if err != nil { + t.Fatalf("failed to read flageval-worker schema fixture: %v", err) + } + + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("batchedflagevaluations.json", strings.NewReader(string(schemaBytes))); err != nil { + t.Fatalf("failed to add flageval-worker schema fixture: %v", err) + } + schema, err := compiler.Compile("batchedflagevaluations.json") + if err != nil { + t.Fatalf("failed to compile flageval-worker schema fixture: %v", err) + } + + b, err := json.Marshal(payload) + if err != nil { + t.Fatalf("failed to marshal payload for schema validation: %v", err) + } + var doc any + if err := json.Unmarshal(b, &doc); err != nil { + t.Fatalf("failed to unmarshal payload for schema validation: %v", err) + } + return schema.Validate(doc) +} + +// TestFlattenAndPruneContextEquivalence verifies the merged single-pass // flattenAndPruneContext must produce a pruned result byte-for-byte identical to the prior // two-step flattenContext + pruneContext pipeline across nested, oversized, and >256-field // inputs (and the determinism + 256/256 limits are preserved). @@ -201,7 +232,7 @@ func TestPruneContext(t *testing.T) { } } -// TestFlagEvaluationPayloadSchema verifies that full, degraded, and ultra-degraded events +// TestFlagEvaluationPayloadSchema verifies that full, degraded, and required-only events // marshal to JSON that omits the expected optional fields per tier while always including // the 5 required fields. // @@ -251,7 +282,7 @@ func TestFlagEvaluationPayloadSchema(t *testing.T) { { name: "required-only event omits all optional fields", // A bare event carrying only flag key + counts; no variant, allocation, targeting, - // context. (This shape is no longer emitted by an ultra-degraded tier, but the + // context. (This shape is not emitted by a dedicated tier, but the // schema must still accept a required-fields-only event.) event: flagEvaluationEvent{ Timestamp: nowMs, @@ -343,6 +374,72 @@ func TestFlagEvaluationPayloadSchema(t *testing.T) { t.Errorf("batch payload: expected 1 flagEvaluations entry, got %v", m["flagEvaluations"]) } }) + + t.Run("batch payload validates against real flageval-worker schema", func(t *testing.T) { + payload := flagEvaluationPayload{ + Context: flagEvalDDContext{ + Service: "test-service", + Env: "test", + Version: "1.0.0", + }, + FlagEvaluations: []flagEvaluationEvent{ + { + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "full-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 2, + RuntimeDefault: true, + TargetingKey: "user-123", + Variant: &flagEvalVariant{Key: "on"}, + Allocation: &flagEvalAllocation{Key: "alloc-a"}, + Error: &flagEvalError{Message: string(of.TypeMismatchCode)}, + Context: &flagEvalEventContext{ + Evaluation: map[string]any{"country": "US", "plan": "pro"}, + }, + }, + { + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "degraded-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 5, + Variant: &flagEvalVariant{Key: "off"}, + Allocation: &flagEvalAllocation{Key: "alloc-b"}, + Error: &flagEvalError{Message: string(of.FlagNotFoundCode)}, + }, + { + Timestamp: nowMs, + Flag: flagEvalFlag{Key: "required-only-flag"}, + FirstEvaluation: nowMs, + LastEvaluation: nowMs, + EvaluationCount: 1, + }, + }, + } + + if err := validateBatchedFlagEvaluationsSchema(t, payload); err != nil { + t.Fatalf("payload should validate against flageval-worker schema: %v", err) + } + }) + + t.Run("worker schema rejects top-level reason", func(t *testing.T) { + payload := map[string]any{ + "context": map[string]any{"service": "test-service"}, + "flagEvaluations": []map[string]any{{ + "timestamp": nowMs, + "flag": map[string]any{"key": "test-flag"}, + "first_evaluation": nowMs, + "last_evaluation": nowMs, + "evaluation_count": 1, + "reason": "targeting_match", + }}, + } + + if err := validateBatchedFlagEvaluationsSchema(t, payload); err == nil { + t.Fatal("payload with top-level reason should be rejected by flageval-worker schema") + } + }) } // TestAggregatorCollisionSafety verifies that two distinct inputs that would collide @@ -360,13 +457,11 @@ func TestAggregatorCollisionSafety(t *testing.T) { flagKey: "my-flag", variant: "on", allocationKey: "alloc-a", - reason: "targeting_match", } d2 := evalDetails{ flagKey: "my-flag", variant: "on", allocationKey: "alloc-b", - reason: "targeting_match", } agg.add(d1, nil, nowMs) @@ -389,6 +484,38 @@ func TestAggregatorCollisionSafety(t *testing.T) { } } +func TestOpenFeatureReasonIsNotEVPCardinality(t *testing.T) { + w := newFlagEvaluationWriter(ProviderConfig{}) + hookCtx := of.NewHookContext( + "reasonless-flag", + of.Boolean, + false, + of.NewClientMetadata(""), + of.Metadata{Name: "test-provider"}, + of.NewEvaluationContext("user-1", map[string]any{"country": "US"}), + ) + metadata := of.FlagMetadata{metadataAllocationKey: "alloc-a"} + detailsA := makeEvalDetails("on", of.TargetingMatchReason, "", metadata) + detailsB := makeEvalDetails("on", of.SplitReason, "", metadata) + + w.record(hookCtx, detailsA) + w.record(hookCtx, detailsB) + for len(w.events) > 0 { + w.aggregate(<-w.events) + } + + w.aggregator.mu.Lock() + defer w.aggregator.mu.Unlock() + if len(w.aggregator.full) != 1 { + t.Fatalf("reason-only differences must not split EVP buckets; got %d full buckets", len(w.aggregator.full)) + } + for _, e := range w.aggregator.full { + if e.count != 2 { + t.Fatalf("reason-only differences should aggregate into count=2, got %d", e.count) + } + } +} + // TestAggregatorConcurrentMinMax verifies that 1000 goroutines recording the same key // produce count==1000 and firstEvaluation<=lastEvaluation. // Must be run with -race to satisfy the race-free requirement. @@ -399,7 +526,6 @@ func TestAggregatorConcurrentMinMax(t *testing.T) { d := evalDetails{ flagKey: "concurrent-flag", variant: "on", - reason: "targeting_match", } const goroutines = 1000 @@ -456,7 +582,6 @@ func TestSaturationCountPreservation(t *testing.T) { flagKey: fmt.Sprintf("sat-flag-%d", flagIdx), variant: "on", allocationKey: fmt.Sprintf("alloc-%d", allocIdx), - reason: "targeting_match", targetingKey: fmt.Sprintf("user-%d", i%10), } agg.add(d, nil, nowMs) @@ -487,7 +612,7 @@ func TestSaturationCountPreservation(t *testing.T) { // TestAggregatorCapOverflow verifies that: // - Exceeding perFlagCap routes new entries to the degraded map. -// - Exceeding degradedCap routes new entries to the ultra-degraded map. +// - Exceeding degradedCap drops new entries and counts the drop. // - Global cap bounds total bucket growth. func TestAggregatorCapOverflow(t *testing.T) { t.Run("perFlagCap overflow routes to degraded", func(t *testing.T) { @@ -500,7 +625,6 @@ func TestAggregatorCapOverflow(t *testing.T) { flagKey: "flag-a", variant: "on", allocationKey: fmt.Sprintf("alloc-%d", i), - reason: "targeting_match", targetingKey: fmt.Sprintf("user-%d", i), } agg.add(d, map[string]any{"key": fmt.Sprintf("v%d", i)}, nowMs) @@ -517,7 +641,6 @@ func TestAggregatorCapOverflow(t *testing.T) { flagKey: "flag-a", variant: "on", allocationKey: "alloc-overflow", - reason: "targeting_match", targetingKey: "user-overflow", } agg.add(d4, map[string]any{"extra": "data"}, nowMs) @@ -542,7 +665,6 @@ func TestAggregatorCapOverflow(t *testing.T) { flagKey: fmt.Sprintf("flag-%d", i), variant: fmt.Sprintf("v%d", j), allocationKey: fmt.Sprintf("alloc-%d", j), - reason: "targeting_match", targetingKey: fmt.Sprintf("user-%d", j), } agg.add(d, nil, nowMs) @@ -555,7 +677,6 @@ func TestAggregatorCapOverflow(t *testing.T) { d := evalDetails{ flagKey: fmt.Sprintf("overflow-flag-%d", i), variant: "on", - reason: "targeting_match", } // Force each into degraded by also filling its full tier for j := range 4 { @@ -563,7 +684,6 @@ func TestAggregatorCapOverflow(t *testing.T) { flagKey: d.flagKey, variant: d.variant, allocationKey: fmt.Sprintf("a%d", j), - reason: d.reason, } agg.add(d2, nil, nowMs) } @@ -593,7 +713,6 @@ func TestAggregatorCapOverflow(t *testing.T) { d := evalDetails{ flagKey: fmt.Sprintf("flag-%d", i), variant: "on", - reason: "targeting_match", } agg.add(d, nil, nowMs) } @@ -624,7 +743,7 @@ func TestAggregatorCapOverflow(t *testing.T) { }) } -// TestPruneContextDeterministic guards Finding #1 (nondeterministic context pruning). +// TestPruneContextDeterministic verifies deterministic context pruning. // A >256-field context must prune to an IDENTICAL kept subset (and therefore an identical // canonical key) on every call, and two independently-built maps with the same logical entries // must produce an equal key. On the pre-fix code (map-range cap BEFORE sort) the kept subset is @@ -654,7 +773,7 @@ func TestPruneContextDeterministic(t *testing.T) { } } -// TestPruneContextOversizedStringDeterministic guards Finding #1 with an oversized-string +// TestPruneContextOversizedStringDeterministic verifies an oversized-string // skip among >256 fields: the oversized-string skip must be applied against a deterministic // key ordering, so the kept subset (and hash) is stable across iterations. func TestPruneContextOversizedStringDeterministic(t *testing.T) { @@ -684,7 +803,7 @@ func TestPruneContextOversizedStringDeterministic(t *testing.T) { } } -// TestCanonicalContextKeyEncoding guards oleksii #2 (comparable canonical-context key replaces +// TestCanonicalContextKeyEncoding verifies that the comparable canonical-context key replaces // the lossy FNV-1a discriminator). Distinct contexts must produce DISTINCT keys — int 1 vs // string "1" must differ, and '='/'\n'-bearing values/keys must not fake a multi-field context. // Because the full canonical encoding IS the map key (no hash), distinct contexts ALWAYS land @@ -739,7 +858,7 @@ func TestCanonicalContextKeyEncoding(t *testing.T) { t.Run("distinct buckets via add(): "+tc.name, func(t *testing.T) { agg := setupTestAggregator(t) nowMs := time.Now().UnixMilli() - d := evalDetails{flagKey: "f", variant: "on", reason: "targeting_match"} + d := evalDetails{flagKey: "f", variant: "on"} agg.add(d, tc.mapA, nowMs) agg.add(d, tc.mapB, nowMs) @@ -762,7 +881,7 @@ func TestCanonicalContextKeyEncoding(t *testing.T) { t.Run("re-adding identical multi-field context increments same bucket", func(t *testing.T) { agg := setupTestAggregator(t) nowMs := time.Now().UnixMilli() - d := evalDetails{flagKey: "f", variant: "on", reason: "targeting_match"} + d := evalDetails{flagKey: "f", variant: "on"} ctx := map[string]any{"a": "x\ny", "b": 7, "c=d": true} agg.add(d, ctx, nowMs) agg.add(d, map[string]any{"b": 7, "a": "x\ny", "c=d": true}, nowMs) // same logical context, different insertion order @@ -780,7 +899,7 @@ func TestCanonicalContextKeyEncoding(t *testing.T) { }) } -// TestDegradedCapBounded guards reviewer concern #8 (unbounded dynamic/abusive flag keys) under +// TestDegradedCapBounded verifies that unbounded dynamic/abusive flag keys stay bounded under // the 2-tier design. With the degraded tier as the terminal tier, an unbounded number of distinct // flag keys must NOT grow the degraded map without bound: len(degraded) <= degradedCap, and the // over-cap counts must be DROPPED-AND-COUNTED (droppedDegradedOverflow), never silently lost. @@ -796,7 +915,6 @@ func TestDegradedCapBounded(t *testing.T) { d := evalDetails{ flagKey: fmt.Sprintf("dynamic-flag-%d", i), // every key distinct variant: "on", - reason: "targeting_match", } agg.add(d, nil, nowMs) } @@ -826,7 +944,7 @@ func TestDegradedCapBounded(t *testing.T) { } } -// TestRecordAfterStopIsNoop guards Finding #4 (record() can enqueue after shutdown). After +// TestRecordAfterStopIsNoop verifies record() cannot enqueue after shutdown. After // stop(), record() must NOT enqueue into the never-drained events channel; the event must be // counted as dropped instead. On the pre-fix code (no stopped check in record()) the event is // enqueued and silently lost. @@ -869,9 +987,49 @@ func TestRecordAfterStopIsNoop(t *testing.T) { } } -// TestExtractEvalDetailsPrefersErrorMessage guards Finding #5 (error payload uses ErrorCode -// instead of OpenFeature's ErrorMessage). ErrorMessage is preferred when present; ErrorCode is -// the fallback only when ErrorMessage is empty. +func TestRecordQueuesPrunedContextSnapshot(t *testing.T) { + w := newFlagEvaluationWriter(ProviderConfig{}) + attrs := make(map[string]any, maxContextFields+50) + for i := range maxContextFields + 50 { + attrs[fmt.Sprintf("field-%03d", i)] = fmt.Sprintf("value-%03d", i) + } + attrs["zzz-oversized"] = strings.Repeat("x", maxFieldLength+1) + hookCtx := of.NewHookContext( + "test-flag", + of.Boolean, + false, + of.NewClientMetadata(""), + of.Metadata{Name: "test-provider"}, + of.NewEvaluationContext("user-1", attrs), + ) + details := of.InterfaceEvaluationDetails{ + Value: true, + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "test-flag", + FlagType: of.Boolean, + ResolutionDetail: of.ResolutionDetail{ + Variant: "on", + Reason: of.TargetingMatchReason, + }, + }, + } + + w.record(hookCtx, details) + + if len(w.events) != 1 { + t.Fatalf("expected one queued event, got %d", len(w.events)) + } + ev := <-w.events + if got := len(ev.contextAttrs); got != maxContextFields { + t.Fatalf("queued context should be pruned to %d fields, got %d", maxContextFields, got) + } + if _, ok := ev.contextAttrs["zzz-oversized"]; ok { + t.Fatal("queued context should not contain oversized string values") + } +} + +// TestExtractEvalDetailsPrefersErrorMessage verifies ErrorMessage is preferred when present; +// ErrorCode is the fallback only when ErrorMessage is empty. func TestExtractEvalDetailsPrefersErrorMessage(t *testing.T) { mkHookCtx := func() of.HookContext { return of.NewHookContext( diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 456c8c0e453..54b2a6a142f 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -413,10 +413,10 @@ func scaleBenchProfile() flagEvalBenchProfile { // BenchmarkFlagEvaluationOTelPlusEVPParallel measures concurrent server-side evaluation // through a real OpenFeature client with BOTH the OTel hook and the new EVP flagevaluation -// hook enabled, at the >=2,500-flag scale profile (oleksii review #5). Because every +// hook enabled, at the >=2,500-flag scale profile. Because every // concurrent evaluation funnels through the EVP writer's single aggregator mutex // (flagEvaluationAggregator.add), this is the bench that surfaces lock contention under -// realistic multi-goroutine server load — exactly what the parallel reviewer concern asks for. +// realistic multi-goroutine server load. // // Each goroutine rotates its own flag key and targeting key across the full cardinality so the // aggregator sees a realistic spread of buckets, not a single hot key. The 24h flush interval diff --git a/openfeature/testdata/flageval-worker/batchedflagevaluations.json b/openfeature/testdata/flageval-worker/batchedflagevaluations.json new file mode 100644 index 00000000000..b3a4b1fda67 --- /dev/null +++ b/openfeature/testdata/flageval-worker/batchedflagevaluations.json @@ -0,0 +1,184 @@ +{ + "title": "BatchedFlagEvaluations", + "type": "object", + "properties": { + "context": { + "title": "InputContextDatadog", + "type": "object", + "properties": { + "geo": { + "type": "object", + "properties": { + "country_iso_code": { "type": "string" }, + "country": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "rum": { + "type": "object", + "properties": { + "application": { + "type": "object", + "properties": { + "id": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "view": { + "type": "object", + "properties": { + "url": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "service": { "type": "string" }, + "version": { "type": "string" }, + "env": { "type": "string" }, + "device": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "brand": { "type": "string" }, + "model": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "os": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "version": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "flagEvaluations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "timestamp": { + "type": "integer", + "description": "The timestamp (milliseconds since epoch) at which the evaluation occurred." + }, + "flag": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "first_evaluation": { + "type": "integer", + "minimum": 1759276800000 + }, + "last_evaluation": { + "type": "integer", + "minimum": 1759276800000 + }, + "evaluation_count": { + "type": "integer", + "minimum": 1 + }, + "runtime_default_used": { "type": "boolean" }, + "targeting_key": { "type": "string" }, + "context": { + "type": "object", + "properties": { + "evaluation": { "type": "object" }, + "dd": { + "type": "object", + "properties": { + "service": { "type": "string" }, + "rum": { + "type": "object", + "properties": { + "application": { + "type": "object", + "properties": { + "id": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "view": { + "type": "object", + "properties": { + "url": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": true + } + }, + "required": [], + "additionalProperties": false + }, + "variant": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "allocation": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "targeting_rule": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "error": { + "type": "object", + "properties": { + "message": { "type": "string" } + }, + "required": ["message"], + "additionalProperties": false + } + }, + "required": [ + "timestamp", + "flag", + "first_evaluation", + "last_evaluation", + "evaluation_count" + ], + "additionalProperties": false + } + } + }, + "required": ["flagEvaluations"], + "additionalProperties": false +} From 2329fca9d2e163b7ed7b9e95d69326308b0bb436 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 03:04:39 -0400 Subject: [PATCH 24/37] chore(openfeature): mark flagevaluation schema validator direct --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0328145f64e..1cea894820b 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/puzpuzpuz/xsync/v3 v3.5.1 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 github.com/tinylib/msgp v1.6.3 @@ -85,7 +86,6 @@ require ( github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/stretchr/objx v0.5.3 // indirect From 273cf238723a0c1688c57a4b2b4f53a5f749ae7d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 11:20:02 -0400 Subject: [PATCH 25/37] openfeature: drop copied flageval schema --- .gitlab/benchmarks/micro/gitlab-ci.yml | 2 +- go.mod | 1 - go.sum | 2 - openfeature/flagevaluation.go | 4 +- openfeature/flagevaluation_test.go | 129 ++++++++---- .../batchedflagevaluations.json | 184 ------------------ 6 files changed, 94 insertions(+), 228 deletions(-) delete mode 100644 openfeature/testdata/flageval-worker/batchedflagevaluations.json diff --git a/.gitlab/benchmarks/micro/gitlab-ci.yml b/.gitlab/benchmarks/micro/gitlab-ci.yml index c1502155392..f0566db139d 100644 --- a/.gitlab/benchmarks/micro/gitlab-ci.yml +++ b/.gitlab/benchmarks/micro/gitlab-ci.yml @@ -78,7 +78,7 @@ microbenchmarks-1: microbenchmarks-2: extends: .microbenchmarks variables: - BENCHMARKS: "BenchmarkStartRequestSpan|BenchmarkStartRequestSpanQueryObfuscation|BenchmarkHttpServeTrace|BenchmarkHttpServeTraceQueryObfuscation|BenchmarkTracerAddSpans|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkOTelApiWithCustomTags|BenchmarkInjectW3C|BenchmarkExtractW3C|BenchmarkPartialFlushing" + BENCHMARKS: "BenchmarkStartRequestSpan|BenchmarkStartRequestSpanQueryObfuscation|BenchmarkHttpServeTrace|BenchmarkHttpServeTraceQueryObfuscation|BenchmarkTracerAddSpans|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkOTelApiWithCustomTags|BenchmarkInjectW3C|BenchmarkExtractW3C|BenchmarkPartialFlushing|BenchmarkFlagEvaluationNoop|BenchmarkFlagEvaluationOTelOnly|BenchmarkFlagEvaluationOTelPlusEVP|BenchmarkFlagEvaluationEVPRecord|BenchmarkFlagEvaluationOTelPlusEVPParallel" CPUS_PER_BENCHMARK: 1 REPETITIONS: 10 GROUP_ID: "group-2" diff --git a/go.mod b/go.mod index 1cea894820b..aa70db14c5c 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/puzpuzpuz/xsync/v3 v3.5.1 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 github.com/tinylib/msgp v1.6.3 diff --git a/go.sum b/go.sum index caa8e8bba26..02f5751071a 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,6 @@ github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVO github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3/go.mod h1:vl5+MqJ1nBINuSsUI2mGgH79UweUT/B5Fy8857PqyyI= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14= github.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 244210de1aa..3cce4d76adc 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -203,7 +203,9 @@ type flagEvalContextDD struct { Service string `json:"service,omitempty"` } -// flagEvaluationPayload matches batchedflagevaluations.json. +// flagEvaluationPayload is the SDK's EVP flagevaluation batch envelope. +// Keep JSON field names aligned with the worker contract; do not vendor the +// worker schema here, because dd-source owns that contract. type flagEvaluationPayload struct { Context flagEvalDDContext `json:"context"` FlagEvaluations []flagEvaluationEvent `json:"flagEvaluations"` diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 0169dbc2f04..659ceac56cd 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -8,44 +8,27 @@ package openfeature import ( "encoding/json" "fmt" - "os" "reflect" "strings" "sync" "testing" "time" - "github.com/santhosh-tekuri/jsonschema/v5" - of "github.com/open-feature/go-sdk/openfeature" ) -func validateBatchedFlagEvaluationsSchema(t *testing.T, payload any) error { +func mustMarshalJSONMap(t *testing.T, payload any) map[string]any { t.Helper() - schemaBytes, err := os.ReadFile("testdata/flageval-worker/batchedflagevaluations.json") - if err != nil { - t.Fatalf("failed to read flageval-worker schema fixture: %v", err) - } - - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("batchedflagevaluations.json", strings.NewReader(string(schemaBytes))); err != nil { - t.Fatalf("failed to add flageval-worker schema fixture: %v", err) - } - schema, err := compiler.Compile("batchedflagevaluations.json") - if err != nil { - t.Fatalf("failed to compile flageval-worker schema fixture: %v", err) - } - b, err := json.Marshal(payload) if err != nil { - t.Fatalf("failed to marshal payload for schema validation: %v", err) + t.Fatalf("failed to marshal payload: %v", err) } - var doc any - if err := json.Unmarshal(b, &doc); err != nil { + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { t.Fatalf("failed to unmarshal payload for schema validation: %v", err) } - return schema.Validate(doc) + return m } // TestFlattenAndPruneContextEquivalence verifies the merged single-pass @@ -375,7 +358,7 @@ func TestFlagEvaluationPayloadSchema(t *testing.T) { } }) - t.Run("batch payload validates against real flageval-worker schema", func(t *testing.T) { + t.Run("batch payload uses only stable EVP flagevaluation fields", func(t *testing.T) { payload := flagEvaluationPayload{ Context: flagEvalDDContext{ Service: "test-service", @@ -418,26 +401,94 @@ func TestFlagEvaluationPayloadSchema(t *testing.T) { }, } - if err := validateBatchedFlagEvaluationsSchema(t, payload); err != nil { - t.Fatalf("payload should validate against flageval-worker schema: %v", err) + m := mustMarshalJSONMap(t, payload) + for k := range m { + switch k { + case "context", "flagEvaluations": + default: + t.Fatalf("unexpected batch payload field %q", k) + } + } + + arr, ok := m["flagEvaluations"].([]any) + if !ok { + t.Fatalf("flagEvaluations is not an array: %T", m["flagEvaluations"]) + } + if len(arr) != 3 { + t.Fatalf("expected 3 flagEvaluations entries, got %d", len(arr)) + } + + allowedEventFields := map[string]struct{}{ + "timestamp": {}, + "flag": {}, + "first_evaluation": {}, + "last_evaluation": {}, + "evaluation_count": {}, + "runtime_default_used": {}, + "targeting_key": {}, + "variant": {}, + "allocation": {}, + "targeting_rule": {}, + "error": {}, + "context": {}, + } + for i, raw := range arr { + event, ok := raw.(map[string]any) + if !ok { + t.Fatalf("flagEvaluations[%d] is not an object: %T", i, raw) + } + for k := range event { + if _, ok := allowedEventFields[k]; !ok { + t.Fatalf("flagEvaluations[%d] emitted unexpected field %q", i, k) + } + } + for _, required := range requiredFields { + if _, ok := event[required]; !ok { + t.Fatalf("flagEvaluations[%d] missing required field %q", i, required) + } + } + flag, ok := event["flag"].(map[string]any) + if !ok { + t.Fatalf("flagEvaluations[%d].flag is not an object: %T", i, event["flag"]) + } + if _, ok := flag["key"]; !ok { + t.Fatalf("flagEvaluations[%d].flag.key missing", i) + } + } + + full := arr[0].(map[string]any) + if _, ok := full["reason"]; ok { + t.Fatal("full EVP event must not emit OpenFeature reason") + } + if _, ok := full["targeting_key"]; !ok { + t.Fatal("full EVP event should retain targeting_key") + } + if _, ok := full["context"]; !ok { + t.Fatal("full EVP event should retain context") } - }) - t.Run("worker schema rejects top-level reason", func(t *testing.T) { - payload := map[string]any{ - "context": map[string]any{"service": "test-service"}, - "flagEvaluations": []map[string]any{{ - "timestamp": nowMs, - "flag": map[string]any{"key": "test-flag"}, - "first_evaluation": nowMs, - "last_evaluation": nowMs, - "evaluation_count": 1, - "reason": "targeting_match", - }}, + degraded := arr[1].(map[string]any) + if _, ok := degraded["reason"]; ok { + t.Fatal("degraded EVP event must not emit OpenFeature reason") + } + if _, ok := degraded["targeting_key"]; ok { + t.Fatal("degraded EVP event should omit targeting_key") + } + if _, ok := degraded["context"]; ok { + t.Fatal("degraded EVP event should omit context") + } + if _, ok := degraded["variant"]; !ok { + t.Fatal("degraded EVP event should retain schema-visible variant") + } + if _, ok := degraded["allocation"]; !ok { + t.Fatal("degraded EVP event should retain schema-visible allocation") } - if err := validateBatchedFlagEvaluationsSchema(t, payload); err == nil { - t.Fatal("payload with top-level reason should be rejected by flageval-worker schema") + requiredOnly := arr[2].(map[string]any) + for _, optional := range []string{"reason", "targeting_key", "variant", "allocation", "targeting_rule", "error", "context", "runtime_default_used"} { + if _, ok := requiredOnly[optional]; ok { + t.Fatalf("required-only EVP event should omit %q", optional) + } } }) } diff --git a/openfeature/testdata/flageval-worker/batchedflagevaluations.json b/openfeature/testdata/flageval-worker/batchedflagevaluations.json deleted file mode 100644 index b3a4b1fda67..00000000000 --- a/openfeature/testdata/flageval-worker/batchedflagevaluations.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "title": "BatchedFlagEvaluations", - "type": "object", - "properties": { - "context": { - "title": "InputContextDatadog", - "type": "object", - "properties": { - "geo": { - "type": "object", - "properties": { - "country_iso_code": { "type": "string" }, - "country": { "type": "string" } - }, - "required": [], - "additionalProperties": false - }, - "rum": { - "type": "object", - "properties": { - "application": { - "type": "object", - "properties": { - "id": { "type": "string" } - }, - "required": [], - "additionalProperties": false - }, - "view": { - "type": "object", - "properties": { - "url": { "type": "string" } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - }, - "service": { "type": "string" }, - "version": { "type": "string" }, - "env": { "type": "string" }, - "device": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "type": { "type": "string" }, - "brand": { "type": "string" }, - "model": { "type": "string" } - }, - "required": [], - "additionalProperties": false - }, - "os": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "version": { "type": "string" } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - }, - "flagEvaluations": { - "type": "array", - "items": { - "type": "object", - "properties": { - "timestamp": { - "type": "integer", - "description": "The timestamp (milliseconds since epoch) at which the evaluation occurred." - }, - "flag": { - "type": "object", - "properties": { - "key": { "type": "string" } - }, - "required": ["key"], - "additionalProperties": false - }, - "first_evaluation": { - "type": "integer", - "minimum": 1759276800000 - }, - "last_evaluation": { - "type": "integer", - "minimum": 1759276800000 - }, - "evaluation_count": { - "type": "integer", - "minimum": 1 - }, - "runtime_default_used": { "type": "boolean" }, - "targeting_key": { "type": "string" }, - "context": { - "type": "object", - "properties": { - "evaluation": { "type": "object" }, - "dd": { - "type": "object", - "properties": { - "service": { "type": "string" }, - "rum": { - "type": "object", - "properties": { - "application": { - "type": "object", - "properties": { - "id": { "type": "string" } - }, - "required": [], - "additionalProperties": false - }, - "view": { - "type": "object", - "properties": { - "url": { "type": "string" } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": true - } - }, - "required": [], - "additionalProperties": false - }, - "variant": { - "type": "object", - "properties": { - "key": { "type": "string" } - }, - "required": ["key"], - "additionalProperties": false - }, - "allocation": { - "type": "object", - "properties": { - "key": { "type": "string" } - }, - "required": ["key"], - "additionalProperties": false - }, - "targeting_rule": { - "type": "object", - "properties": { - "key": { "type": "string" } - }, - "required": ["key"], - "additionalProperties": false - }, - "error": { - "type": "object", - "properties": { - "message": { "type": "string" } - }, - "required": ["message"], - "additionalProperties": false - } - }, - "required": [ - "timestamp", - "flag", - "first_evaluation", - "last_evaluation", - "evaluation_count" - ], - "additionalProperties": false - } - } - }, - "required": ["flagEvaluations"], - "additionalProperties": false -} From b2a93e4fb4ff74e7cf7572553b7faeb0fc20ac20 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 11:51:05 -0400 Subject: [PATCH 26/37] ci: isolate new flagevaluation benchmarks --- .gitlab/benchmarks/micro/gitlab-ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitlab/benchmarks/micro/gitlab-ci.yml b/.gitlab/benchmarks/micro/gitlab-ci.yml index f0566db139d..082dfffd208 100644 --- a/.gitlab/benchmarks/micro/gitlab-ci.yml +++ b/.gitlab/benchmarks/micro/gitlab-ci.yml @@ -78,11 +78,19 @@ microbenchmarks-1: microbenchmarks-2: extends: .microbenchmarks variables: - BENCHMARKS: "BenchmarkStartRequestSpan|BenchmarkStartRequestSpanQueryObfuscation|BenchmarkHttpServeTrace|BenchmarkHttpServeTraceQueryObfuscation|BenchmarkTracerAddSpans|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkOTelApiWithCustomTags|BenchmarkInjectW3C|BenchmarkExtractW3C|BenchmarkPartialFlushing|BenchmarkFlagEvaluationNoop|BenchmarkFlagEvaluationOTelOnly|BenchmarkFlagEvaluationOTelPlusEVP|BenchmarkFlagEvaluationEVPRecord|BenchmarkFlagEvaluationOTelPlusEVPParallel" + BENCHMARKS: "BenchmarkStartRequestSpan|BenchmarkStartRequestSpanQueryObfuscation|BenchmarkHttpServeTrace|BenchmarkHttpServeTraceQueryObfuscation|BenchmarkTracerAddSpans|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkOTelApiWithCustomTags|BenchmarkInjectW3C|BenchmarkExtractW3C|BenchmarkPartialFlushing" CPUS_PER_BENCHMARK: 1 REPETITIONS: 10 GROUP_ID: "group-2" +microbenchmarks-flagevaluation: + extends: .microbenchmarks + variables: + BENCHMARKS: "BenchmarkFlagEvaluationNoop|BenchmarkFlagEvaluationOTelOnly|BenchmarkFlagEvaluationOTelPlusEVP|BenchmarkFlagEvaluationEVPRecord|BenchmarkFlagEvaluationOTelPlusEVPParallel" + CPUS_PER_BENCHMARK: 1 + REPETITIONS: 10 + GROUP_ID: "group-flagevaluation" + pr-performance-gates: stage: microbenchmarks needs: [microbenchmarks-1, microbenchmarks-2] From 5e9b8b014cb6d4ff27fe1278f42bf584959fb38b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 12:39:14 -0400 Subject: [PATCH 27/37] refactor: share openfeature evp transport --- openfeature/evp.go | 81 +++++++++++++++++ openfeature/exposure.go | 67 ++------------ openfeature/flagevaluation.go | 60 ++----------- openfeature/flagevaluation_hook.go | 2 +- openfeature/flagevaluation_hook_test.go | 4 - openfeature/flagevaluation_provider_test.go | 4 + openfeature/flagevaluation_scale_test.go | 97 +++++++++------------ openfeature/flagevaluation_test.go | 13 +-- openfeature/provider.go | 6 +- openfeature/provider_bench_test.go | 3 +- 10 files changed, 147 insertions(+), 190 deletions(-) create mode 100644 openfeature/evp.go diff --git a/openfeature/evp.go b/openfeature/evp.go new file mode 100644 index 00000000000..c13433a8849 --- /dev/null +++ b/openfeature/evp.go @@ -0,0 +1,81 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +package openfeature + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + + jsoniter "github.com/json-iterator/go" + + "github.com/DataDog/dd-trace-go/v2/internal" + "github.com/DataDog/dd-trace-go/v2/internal/log" +) + +type evpClient struct { + httpClient *http.Client + agentURL *url.URL + jsonConfig jsoniter.API +} + +func newEVPClient() *evpClient { + agentURL := internal.AgentURLFromEnv() + var httpClient *http.Client + if agentURL.Scheme == "unix" { + httpClient = internal.UDSClient(agentURL.Path, defaultHTTPTimeout) + agentURL = internal.UnixDataSocketURL(agentURL.Path) + } else { + httpClient = internal.DefaultHTTPClient(defaultHTTPTimeout, false) + } + + return &evpClient{ + httpClient: httpClient, + agentURL: agentURL, + jsonConfig: jsoniter.Config{}.Froze(), + } +} + +func (c *evpClient) post(endpoint, eventName string, payload any) error { + if c == nil { + return fmt.Errorf("EVP client is not configured") + } + + var bytesBuffer bytes.Buffer + encoder := c.jsonConfig.NewEncoder(&bytesBuffer) + if err := encoder.Encode(payload); err != nil { + return fmt.Errorf("failed to encode %s payload: %w", eventName, err) + } + + u := *c.agentURL + u.Path = endpoint + requestURL := u.String() + + req, err := http.NewRequestWithContext(context.Background(), "POST", requestURL, &bytesBuffer) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set(evpSubdomainHeader, evpSubdomainValue) + + log.Debug("openfeature: sending %s events to %s", eventName, requestURL) + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 256)) + return fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) + } + return nil +} diff --git a/openfeature/exposure.go b/openfeature/exposure.go index cc1df0fef8c..ccd46b76bb8 100644 --- a/openfeature/exposure.go +++ b/openfeature/exposure.go @@ -6,22 +6,13 @@ package openfeature import ( - "bytes" "cmp" "container/list" - "context" - "fmt" - "io" "log/slog" - "net/http" - "net/url" "os" "sync" "time" - jsoniter "github.com/json-iterator/go" - - "github.com/DataDog/dd-trace-go/v2/internal" "github.com/DataDog/dd-trace-go/v2/internal/env" "github.com/DataDog/dd-trace-go/v2/internal/globalconfig" "github.com/DataDog/dd-trace-go/v2/internal/log" @@ -176,36 +167,27 @@ type exposureWriter struct { buffer []exposureEvent // Buffer for exposure events cache *exposureLRUCache // LRU deduplication cache flushInterval time.Duration - httpClient *http.Client - agentURL *url.URL + evp *evpClient context exposureContext ticker *time.Ticker stopChan chan struct{} stopped bool - jsonConfig jsoniter.API } // newExposureWriter creates a new exposure writer with the given configuration func newExposureWriter(config ProviderConfig) *exposureWriter { - agentURL := internal.AgentURLFromEnv() - var httpClient *http.Client - if agentURL.Scheme == "unix" { - httpClient = internal.UDSClient(agentURL.Path, defaultHTTPTimeout) - agentURL = internal.UnixDataSocketURL(agentURL.Path) - } else { - httpClient = internal.DefaultHTTPClient(defaultHTTPTimeout, false) - } + return newExposureWriterWithEVP(config, newEVPClient()) +} +func newExposureWriterWithEVP(config ProviderConfig, evp *evpClient) *exposureWriter { executable, _ := os.Executable() return &exposureWriter{ buffer: make([]exposureEvent, 0, 1<<8), // Initial capacity of 256 cache: newExposureLRUCache(defaultExposureCacheCapacity), flushInterval: cmp.Or(config.ExposureFlushInterval, defaultExposureFlushInterval), - httpClient: httpClient, - agentURL: agentURL, + evp: evp, stopChan: make(chan struct{}), - jsonConfig: jsoniter.Config{}.Froze(), context: exposureContext{ Service: cmp.Or(env.Get("DD_SERVICE"), globalconfig.ServiceName(), executable), Version: env.Get("DD_VERSION"), @@ -298,44 +280,7 @@ func (w *exposureWriter) flush() { // sendToAgent sends the exposure payload to the Datadog Agent via EVP proxy func (w *exposureWriter) sendToAgent(payload exposurePayload) error { - // Serialize payload - var bytesBuffer bytes.Buffer - encoder := w.jsonConfig.NewEncoder(&bytesBuffer) - if err := encoder.Encode(payload); err != nil { - return fmt.Errorf("failed to encode exposure payload: %w", err) - } - - // Build request URL - u := *w.agentURL - u.Path = exposureEndpoint - requestURL := u.String() - - // Create HTTP request - req, err := http.NewRequestWithContext(context.Background(), "POST", requestURL, &bytesBuffer) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - // Set headers - req.Header.Set("Content-Type", "application/json") - req.Header.Set(evpSubdomainHeader, evpSubdomainValue) - - log.Debug("openfeature: sending exposure events to %s", requestURL) - - // Send request - resp, err := w.httpClient.Do(req) - if err != nil { - return fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() - - // Check response status - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 256)) - return fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) - } - - return nil + return w.evp.post(exposureEndpoint, "exposure", payload) } // stop stops the exposure writer and flushes any remaining events diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 3cce4d76adc..973af612bc7 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -6,14 +6,9 @@ package openfeature import ( - "bytes" "cmp" - "context" "fmt" - "io" "log/slog" - "net/http" - "net/url" "os" "sort" "strconv" @@ -21,9 +16,6 @@ import ( "sync/atomic" "time" - jsoniter "github.com/json-iterator/go" - - "github.com/DataDog/dd-trace-go/v2/internal" "github.com/DataDog/dd-trace-go/v2/internal/env" "github.com/DataDog/dd-trace-go/v2/internal/globalconfig" "github.com/DataDog/dd-trace-go/v2/internal/log" @@ -222,13 +214,11 @@ type flagEvalDDContext struct { type flagEvaluationWriter struct { aggregator flagEvaluationAggregator flushInterval time.Duration - httpClient *http.Client - agentURL *url.URL + evp *evpClient ddContext flagEvalDDContext // service/env/version — same source as exposureContext ticker *time.Ticker stopChan chan struct{} stopped atomic.Bool // single idempotency gate for stop(); also read lock-free by record() - jsonConfig jsoniter.API // Asynchronous hand-off: the Finally hook enqueues a bounded snapshot here; a single // background worker (started in start()) drains it and performs aggregate/flush off the @@ -264,29 +254,21 @@ type evalDetails struct { } // newFlagEvaluationWriter creates a new flag evaluation writer. -// The writer uses the same HTTP transport setup as exposure.go. func newFlagEvaluationWriter(config ProviderConfig) *flagEvaluationWriter { - agentURL := internal.AgentURLFromEnv() - var httpClient *http.Client - if agentURL.Scheme == "unix" { - httpClient = internal.UDSClient(agentURL.Path, defaultHTTPTimeout) - agentURL = internal.UnixDataSocketURL(agentURL.Path) - } else { - httpClient = internal.DefaultHTTPClient(defaultHTTPTimeout, false) - } + return newFlagEvaluationWriterWithEVP(config, newEVPClient()) +} +func newFlagEvaluationWriterWithEVP(config ProviderConfig, evp *evpClient) *flagEvaluationWriter { executable, _ := os.Executable() flushInterval := cmp.Or(config.FlagEvaluationFlushInterval, defaultFlagEvalFlushInterval) return &flagEvaluationWriter{ flushInterval: flushInterval, - httpClient: httpClient, - agentURL: agentURL, + evp: evp, stopChan: make(chan struct{}), workerDone: make(chan struct{}), events: make(chan evalEvent, defaultEvalEventBufferSize), - jsonConfig: jsoniter.Config{}.Froze(), ddContext: flagEvalDDContext{ Service: cmp.Or(env.Get("DD_SERVICE"), globalconfig.ServiceName(), executable), Version: env.Get("DD_VERSION"), @@ -604,37 +586,7 @@ func (w *flagEvaluationWriter) drainAndFlush() { // sendToAgent sends the flag evaluation payload to the Datadog Agent via EVP proxy. // Reuses evpSubdomainHeader / evpSubdomainValue constants from exposure.go. func (w *flagEvaluationWriter) sendToAgent(payload flagEvaluationPayload) error { - var bytesBuffer bytes.Buffer - encoder := w.jsonConfig.NewEncoder(&bytesBuffer) - if err := encoder.Encode(payload); err != nil { - return fmt.Errorf("failed to encode flag evaluation payload: %w", err) - } - - u := *w.agentURL - u.Path = flagEvaluationEndpoint - requestURL := u.String() - - req, err := http.NewRequestWithContext(context.Background(), "POST", requestURL, &bytesBuffer) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set(evpSubdomainHeader, evpSubdomainValue) - - log.Debug("openfeature: sending flag evaluation events to %s", requestURL) - - resp, err := w.httpClient.Do(req) - if err != nil { - return fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 256)) - return fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) - } - return nil + return w.evp.post(flagEvaluationEndpoint, "flag evaluation", payload) } // add records one evaluation observation into the appropriate aggregation tier. diff --git a/openfeature/flagevaluation_hook.go b/openfeature/flagevaluation_hook.go index de1054cd924..f6b169d724c 100644 --- a/openfeature/flagevaluation_hook.go +++ b/openfeature/flagevaluation_hook.go @@ -44,7 +44,7 @@ func (h *flagEvaluationHook) Finally( h.writer.record(hookContext, details) } -// isRuntimeDefault returns true when the caller's supplied default value was returned. +// isRuntimeDefault returns true when this provider path returns the caller's supplied default. // The primary signal is an absent variant key. Type mismatches are the exception: the provider // may have produced a real variant, but the OpenFeature SDK returns the caller default after // conversion fails. diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index f9dfe16e887..48367488977 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - jsoniter "github.com/json-iterator/go" of "github.com/open-feature/go-sdk/openfeature" ) @@ -20,7 +19,6 @@ func setupTestWriter(t *testing.T) *flagEvaluationWriter { t.Helper() return &flagEvaluationWriter{ flushInterval: 24 * time.Hour, // effectively disabled; tests control flush manually - jsonConfig: jsoniter.Config{}.Froze(), stopChan: make(chan struct{}), events: make(chan evalEvent, defaultEvalEventBufferSize), aggregator: flagEvaluationAggregator{ @@ -206,8 +204,6 @@ func TestRecordUsesEvalTimeFromMetadata(t *testing.T) { // TestFlagEvaluationHookFinally verifies that the Finally hook records an entry for // success, error-reason, and provider-not-ready paths. -// -// It must fail RED: the hook's Finally method panics with "not implemented". func TestFlagEvaluationHookFinally(t *testing.T) { runtimeDefaultTrue := true diff --git a/openfeature/flagevaluation_provider_test.go b/openfeature/flagevaluation_provider_test.go index 3b81fd12cab..d4b2f594238 100644 --- a/openfeature/flagevaluation_provider_test.go +++ b/openfeature/flagevaluation_provider_test.go @@ -89,6 +89,10 @@ func TestFlagEvaluationKillswitch(t *testing.T) { t.Errorf("expected EVP flagEvaluationHook to be absent from Hooks() when killswitch is disabled, but found one") } } + + if tc.wantEVPEnabled && p.exposureWriter.evp != p.flagEvalWriter.evp { + t.Error("expected exposures and flag evaluations to share one EVP client") + } }) } } diff --git a/openfeature/flagevaluation_scale_test.go b/openfeature/flagevaluation_scale_test.go index cc162d7aaf2..07a15d95242 100644 --- a/openfeature/flagevaluation_scale_test.go +++ b/openfeature/flagevaluation_scale_test.go @@ -11,17 +11,8 @@ import ( "time" ) -// These tests drive flagEvaluationAggregator.add directly (no client, no hooks, no async -// worker) to deterministically simulate the team's >=2,500-flag scale target with realistic -// flag structure, then inspect the two aggregation maps. The goal is to answer: -// -// 1. Does 2,500 flags' worth of legitimate schema-visible degraded buckets fit under degradedCap, -// or does it overflow and drop under a 2-tier design? -// 2. What degradedCap/globalCap values make a 2-tier design never drop legitimate counts at -// 2,500 flags? -// -// They are not assertions about correct behavior so much as a measurement harness; each test -// logs its numbers via t.Logf and is intended to be read with `go test -run TestScale -v`. +// These tests drive flagEvaluationAggregator.add directly (no client, no hooks, no async worker) +// to assert the 2,500-flag scale target against production aggregation caps. // scaleFlagShape describes the realistic per-flag structure used to size the degraded tier. // Degraded key includes schema-visible retained fields only. OpenFeature reason is not accepted @@ -132,9 +123,8 @@ func snapshot(agg *flagEvaluationAggregator) tierCounts { return tc } -// TestScaleDegradedCardinality2500Flags reports the LEGITIMATE degraded cardinality for the -// realistic 2,500-flag shape and compares it to the production degradedCap (10,000). This is the -// cap-sizing math that decides whether a 2-tier design drops legitimate counts. +// TestScaleDegradedCardinality2500Flags verifies the production degradedCap has at least 2x +// headroom over the legitimate degraded cardinality of the 2,500-flag target shape. func TestScaleDegradedCardinality2500Flags(t *testing.T) { const n = 2500 flags := makeScaleFlags(n) @@ -160,19 +150,13 @@ func TestScaleDegradedCardinality2500Flags(t *testing.T) { t.Logf("production degradedCap = %d", defaultEvalDegradedCap) t.Logf("production globalCap = %d", defaultEvalGlobalCap) - if deg > defaultEvalDegradedCap { - t.Logf("RESULT: legitimate degraded cardinality %d EXCEEDS degradedCap %d by %d "+ - "=> under a 2-tier design these would DROP. degradedCap must be raised.", - deg, defaultEvalDegradedCap, deg-defaultEvalDegradedCap) - } else { - t.Logf("RESULT: legitimate degraded cardinality %d FITS under degradedCap %d (headroom %d).", - deg, defaultEvalDegradedCap, defaultEvalDegradedCap-deg) - } - - // Recommendation math: degraded must hold full legitimate cardinality; global (full tier) - // must hold flags × contexts up to a reasonable subject bound. Report a 2x-headroom rec. recDegraded := roundUpTo(deg*2, 1000) - t.Logf("RECOMMENDATION: degradedCap >= %d (2x headroom over %d legitimate buckets).", recDegraded, deg) + if defaultEvalDegradedCap < recDegraded { + t.Fatalf("degradedCap=%d does not provide 2x headroom over legitimate cardinality=%d; want >= %d", + defaultEvalDegradedCap, deg, recDegraded) + } + t.Logf("degradedCap headroom OK: cap=%d legitimate=%d requiredWith2xHeadroom=%d", + defaultEvalDegradedCap, deg, recDegraded) } func roundUpTo(v, mult int) int { @@ -182,10 +166,8 @@ func roundUpTo(v, mult int) int { return ((v / mult) + 1) * mult } -// TestScaleDropTriggerSweep is the decisive test for the 2-tier design: across a -// context-cardinality sweep at 2,500 flags, it reports whether the terminal-tier DROP -// (droppedDegradedOverflow) ever fires, and under what conditions. It runs each sweep point with -// the (resized) production caps. +// TestScaleDropTriggerSweep verifies that the 2,500-flag target shape does not drop legitimate +// counts across representative context-cardinality points with production caps. func TestScaleDropTriggerSweep(t *testing.T) { const n = 2500 flags := makeScaleFlags(n) @@ -229,21 +211,16 @@ func TestScaleDropTriggerSweep(t *testing.T) { t.Errorf("count preservation violated: Σ=%d != calls=%d", tc.sumCounts, calls) } - if tc.dropped > 0 { - t.Logf(" DROP TRIGGERED: %d evaluation(s) dropped (degraded saturated at %d).", - tc.dropped, defaultEvalDegradedCap) - } else { - t.Logf(" DROP NOT TRIGGERED at this sweep point — no legitimate count lost.") + if tc.dropped != 0 { + t.Errorf("unexpected degraded overflow drops at %s: got %d", sp.name, tc.dropped) } }) } } // TestScaleDropRequiresDegradedSaturation isolates the precondition for a terminal-tier drop: -// the degraded tier must be FULL. With the resized degradedCap and realistic 2,500-flag -// structure, it forces the worst case (globalCap=0, everything cascades to degraded) and reports -// whether legitimate degraded cardinality fits under degradedCap — i.e. whether a 2-tier design -// drops any legitimate counts at the scale target. +// the degraded tier must be full. With production degradedCap and realistic 2,500-flag structure, +// it forces all buckets through the degraded tier and asserts that no legitimate counts drop. func TestScaleDropRequiresDegradedSaturation(t *testing.T) { const n = 2500 flags := makeScaleFlags(n) @@ -277,10 +254,8 @@ func TestScaleDropRequiresDegradedSaturation(t *testing.T) { } } -// TestScaleFullSaturationCascade saturates the FULL tier naturally (production caps) with -// 2,500 flags × enough distinct subjects that globalCount exceeds globalCap, then reports which -// tier absorbs the overflow. In the 2-tier design, global-cap overflow cascades to degraded -// (which is sized to hold the legitimate degraded cardinality) before any drop. +// TestScaleFullSaturationCascade saturates the full tier naturally with production caps and +// verifies that overflow cascades to degraded without dropping legitimate counts. func TestScaleFullSaturationCascade(t *testing.T) { const n = 2500 flags := makeScaleFlags(n) @@ -291,12 +266,12 @@ func TestScaleFullSaturationCascade(t *testing.T) { defaultEvalPerFlagCap, defaultEvalDegradedCap, ) - // legitimate degraded cardinality is the schema-visible combo count; with enough distinct subjects per combo the - // full tier sees ~173k distinct full keys, over globalCap, forcing overflow into degraded. - calls := driveScale(agg, flags, 1_000_000, 8) + // With 16 distinct subjects per combo the full tier sees ~173k distinct full keys, over + // globalCap, forcing overflow into degraded. + calls := driveScale(agg, flags, 1_000_000, 16) tc := snapshot(agg) - t.Logf("FULL-saturation cascade (production caps; 8 subjects/combo):") + t.Logf("FULL-saturation cascade (production caps; 16 subjects/combo):") t.Logf(" legitimate degraded cardinality=%d add() calls=%d", deg, calls) t.Logf(" full=%d (globalCount=%d, cap=%d) degraded=%d (cap=%d) droppedDegradedOverflow=%d", tc.full, tc.globalCount, defaultEvalGlobalCap, @@ -306,17 +281,22 @@ func TestScaleFullSaturationCascade(t *testing.T) { if tc.full > defaultEvalGlobalCap { t.Errorf("full tier %d exceeded globalCap %d", tc.full, defaultEvalGlobalCap) } + if tc.globalCount != defaultEvalGlobalCap { + t.Errorf("full tier did not saturate: globalCount=%d, want %d", tc.globalCount, defaultEvalGlobalCap) + } + if tc.degraded == 0 { + t.Error("expected degraded tier to absorb full-tier overflow") + } + if tc.dropped != 0 { + t.Errorf("unexpected degraded overflow drops: got %d", tc.dropped) + } if tc.sumCounts != calls { t.Errorf("count preservation violated: Σ=%d != calls=%d", tc.sumCounts, calls) } - t.Logf(" STRUCTURAL NOTE: global-cap overflow cascades to degraded (degraded=%d). "+ - "A drop only occurs once degraded itself reaches degradedCap=%d.", - tc.degraded, defaultEvalDegradedCap) } -// TestScaleHotFlagPerFlagCap drives a SINGLE flag past perFlagCap (10,000 distinct full -// buckets) to demonstrate the per-flag overflow path into the degraded tier, and reports whether -// that fill can in turn saturate degradedCap and trigger a terminal-tier drop. +// TestScaleHotFlagPerFlagCap drives a single flag past perFlagCap and asserts that overflow +// reaches the degraded tier, then becomes counted drops once degradedCap is saturated. func TestScaleHotFlagPerFlagCap(t *testing.T) { agg := newTestAggregator( defaultEvalGlobalCap, @@ -346,9 +326,14 @@ func TestScaleHotFlagPerFlagCap(t *testing.T) { t.Logf(" full=%d (perFlagCap=%d) degraded=%d (cap=%d) droppedDegradedOverflow=%d", tc.full, defaultEvalPerFlagCap, tc.degraded, defaultEvalDegradedCap, tc.dropped) t.Logf(" Σ counts=%d / calls=%d (preserved=%v)", tc.sumCounts, calls, tc.sumCounts == calls) - if tc.dropped > 0 { - t.Logf(" DROP TRIGGERED via a single abusive/hot flag (degraded saturated at %d). "+ - "This is exactly the abuse case the drop counter must make observable.", tc.degraded) + if tc.full != defaultEvalPerFlagCap { + t.Errorf("full tier did not stop at perFlagCap: full=%d, want %d", tc.full, defaultEvalPerFlagCap) + } + if tc.degraded != defaultEvalDegradedCap { + t.Errorf("degraded tier did not stop at degradedCap: degraded=%d, want %d", tc.degraded, defaultEvalDegradedCap) + } + if tc.dropped == 0 { + t.Error("expected counted drops after single hot flag saturates degradedCap") } if tc.sumCounts != calls { t.Errorf("count preservation violated: Σ=%d != calls=%d", tc.sumCounts, calls) diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 659ceac56cd..7daa8b7e696 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -135,8 +135,6 @@ func newTestAggregator(globalCap, perFlagCap, degradedCap int) *flagEvaluationAg // TestPruneContext verifies that pruneContext applies the 256-field / 256-char limits // before evaluation context enters the aggregation buffer. -// -// It must fail RED: pruneContext panics with "not implemented". func TestPruneContext(t *testing.T) { tests := []struct { name string @@ -218,8 +216,6 @@ func TestPruneContext(t *testing.T) { // TestFlagEvaluationPayloadSchema verifies that full, degraded, and required-only events // marshal to JSON that omits the expected optional fields per tier while always including // the 5 required fields. -// -// It must fail RED: the aggregator.add method panics with "not implemented". func TestFlagEvaluationPayloadSchema(t *testing.T) { nowMs := time.Now().UnixMilli() @@ -493,17 +489,12 @@ func TestFlagEvaluationPayloadSchema(t *testing.T) { }) } -// TestAggregatorCollisionSafety verifies that two distinct inputs that would collide -// under FNV-1a-only map keying land in SEPARATE buckets under the struct-keyed map. -// -// It must fail RED: aggregator.add panics with "not implemented". -func TestAggregatorCollisionSafety(t *testing.T) { +// TestAggregatorDistinctAllocationBuckets verifies that allocation is part of the aggregation key. +func TestAggregatorDistinctAllocationBuckets(t *testing.T) { agg := setupTestAggregator(t) nowMs := time.Now().UnixMilli() // Two evaluations that differ only in allocationKey — they must be in separate full buckets. - // Under FNV-1a alone (on a concatenated string), carefully crafted keys can collide; - // under a struct-keyed map, these are structurally distinct and cannot collide. d1 := evalDetails{ flagKey: "my-flag", variant: "on", diff --git a/openfeature/provider.go b/openfeature/provider.go index 3f79537cf25..30bf8ba7378 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -98,8 +98,10 @@ func NewDatadogProvider(config ProviderConfig) (openfeature.FeatureProvider, err } func newDatadogProvider(config ProviderConfig) *DatadogProvider { + evp := newEVPClient() + // Create exposure writer - writer := newExposureWriter(config) + writer := newExposureWriterWithEVP(config, evp) // Create exposure hook hook := newExposureHook(writer) @@ -124,7 +126,7 @@ func newDatadogProvider(config ProviderConfig) *DatadogProvider { // When false, both fields are left nil and the EVP path is disabled. // The OTel hook (flagEvalHook above) is registered unconditionally. if internal.BoolEnv(flagEvalCountsEnabledEnvVar, true) { - evalWriter := newFlagEvaluationWriter(config) + evalWriter := newFlagEvaluationWriterWithEVP(config, evp) p.flagEvalWriter = evalWriter p.flagEvalEVPHook = newFlagEvaluationHook(evalWriter) } diff --git a/openfeature/provider_bench_test.go b/openfeature/provider_bench_test.go index 54b2a6a142f..b8636070d99 100644 --- a/openfeature/provider_bench_test.go +++ b/openfeature/provider_bench_test.go @@ -139,7 +139,8 @@ func BenchmarkEvaluationWithVaryingFlagCounts(b *testing.B) { provider := newDatadogProvider(ProviderConfig{}) config := createTestConfig() - // Add additional flags with unique monotonic keys so cardinality is exercised. + // Add flags that the benchmark below rotates through, so this case measures + // the configured flag count instead of repeatedly evaluating one key. for i := len(config.Flags); i < count.numFlags; i++ { flagKey := "flag-" + strconv.Itoa(i) config.Flags[flagKey] = &flag{ From 5b75720150bd47a4cfa1f9b8fa19ae4aab1bb6e6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 17 Jun 2026 11:38:35 -0400 Subject: [PATCH 28/37] Disambiguate flag evaluation context fallback types --- openfeature/flagevaluation.go | 7 ++++--- openfeature/flagevaluation_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 973af612bc7..8d96df0d02a 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -753,8 +753,9 @@ func appendLengthDelimited(buf, b []byte) []byte { // appendContextValue appends a CANONICAL, length-delimited rendering of v to buf: a type-tag // byte (distinct per Go type) followed by a length-delimited rendered value. This avoids -// allocation for the common scalar types; rare/complex types fall back to fmt. The encoding -// only needs to be deterministic within a run and collision-free across distinct values. +// allocation for the common scalar types; rare/complex types fall back to a type-qualified +// fmt rendering. The encoding only needs to be deterministic within a run and collision-free +// across distinct values. func appendContextValue(buf []byte, v any) []byte { var scratch [32]byte tmp := scratch[:0] @@ -783,7 +784,7 @@ func appendContextValue(buf []byte, v any) []byte { tmp = strconv.AppendFloat(tmp, float64(x), 'g', -1, 32) default: tag = ctxTagOther - tmp = append(tmp, fmt.Sprintf("%v", x)...) + tmp = fmt.Appendf(tmp, "%T:%v", x, x) } buf = append(buf, tag) return appendLengthDelimited(buf, tmp) diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 7daa8b7e696..9e9d33715b9 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -885,6 +885,35 @@ func TestCanonicalContextKeyEncoding(t *testing.T) { }) } + t.Run("supported numeric scalar types produce distinct keys", func(t *testing.T) { + values := []struct { + name string + value any + }{ + {name: "int", value: int(1)}, + {name: "int8", value: int8(1)}, + {name: "int16", value: int16(1)}, + {name: "int32", value: int32(1)}, + {name: "int64", value: int64(1)}, + {name: "uint", value: uint(1)}, + {name: "uint8", value: uint8(1)}, + {name: "uint16", value: uint16(1)}, + {name: "uint32", value: uint32(1)}, + {name: "uint64", value: uint64(1)}, + {name: "float32", value: float32(1)}, + {name: "float64", value: float64(1)}, + } + + keys := make(map[string]string, len(values)) + for _, item := range values { + key := canonicalContextKey(map[string]any{"x": item.value}) + if other, ok := keys[key]; ok { + t.Fatalf("%s and %s produced the same canonical context key", item.name, other) + } + keys[key] = item.name + } + }) + // Logically-identical contexts must produce the SAME key (so they aggregate into one bucket). t.Run("identical contexts produce identical keys", func(t *testing.T) { a := canonicalContextKey(map[string]any{"x": 1, "y": "two"}) From 924712fa36c46e9a0f14e41e23a335d6ab249e64 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 17 Jun 2026 14:50:31 -0400 Subject: [PATCH 29/37] Address flag evaluation benchmark review feedback --- .gitlab/benchmarks/micro/gitlab-ci.yml | 6 +-- openfeature/flagevaluation_test.go | 75 ++++++++++++++++++++++++++ openfeature/provider.go | 1 - 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/.gitlab/benchmarks/micro/gitlab-ci.yml b/.gitlab/benchmarks/micro/gitlab-ci.yml index 082dfffd208..3224598f6dd 100644 --- a/.gitlab/benchmarks/micro/gitlab-ci.yml +++ b/.gitlab/benchmarks/micro/gitlab-ci.yml @@ -83,17 +83,17 @@ microbenchmarks-2: REPETITIONS: 10 GROUP_ID: "group-2" -microbenchmarks-flagevaluation: +microbenchmarks-3: extends: .microbenchmarks variables: BENCHMARKS: "BenchmarkFlagEvaluationNoop|BenchmarkFlagEvaluationOTelOnly|BenchmarkFlagEvaluationOTelPlusEVP|BenchmarkFlagEvaluationEVPRecord|BenchmarkFlagEvaluationOTelPlusEVPParallel" CPUS_PER_BENCHMARK: 1 REPETITIONS: 10 - GROUP_ID: "group-flagevaluation" + GROUP_ID: "group-3" pr-performance-gates: stage: microbenchmarks - needs: [microbenchmarks-1, microbenchmarks-2] + needs: [microbenchmarks-1, microbenchmarks-2, microbenchmarks-3] rules: *benchmark-rules tags: - "arch:amd64" diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 9e9d33715b9..1ed10603754 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -8,6 +8,9 @@ package openfeature import ( "encoding/json" "fmt" + "net/http" + "net/http/httptest" + "net/url" "reflect" "strings" "sync" @@ -1058,6 +1061,78 @@ func TestRecordAfterStopIsNoop(t *testing.T) { } } +func TestStopDrainsAndFlushesQueuedFlagEvaluations(t *testing.T) { + payloads := make(chan flagEvaluationPayload, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + + if r.URL.Path != flagEvaluationEndpoint { + t.Errorf("unexpected EVP path: got %q, want %q", r.URL.Path, flagEvaluationEndpoint) + } + + var payload flagEvaluationPayload + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Errorf("failed to decode flagevaluation payload: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + payloads <- payload + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + + agentURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("failed to parse test server URL: %v", err) + } + + evp := newEVPClient() + evp.agentURL = agentURL + evp.httpClient = server.Client() + + w := newFlagEvaluationWriterWithEVP(ProviderConfig{FlagEvaluationFlushInterval: time.Hour}, evp) + w.start() + + hookCtx := of.NewHookContext( + "test-flag", + of.Boolean, + false, + of.NewClientMetadata(""), + of.Metadata{Name: "test-provider"}, + of.NewEvaluationContext("user-1", map[string]any{"country": "US"}), + ) + details := of.InterfaceEvaluationDetails{ + Value: true, + EvaluationDetails: of.EvaluationDetails{ + FlagKey: "test-flag", + FlagType: of.Boolean, + ResolutionDetail: of.ResolutionDetail{ + Variant: "on", + Reason: of.TargetingMatchReason, + }, + }, + } + w.record(hookCtx, details) + + w.stop() + + select { + case payload := <-payloads: + if len(payload.FlagEvaluations) != 1 { + t.Fatalf("expected 1 flushed flagevaluation event, got %d", len(payload.FlagEvaluations)) + } + event := payload.FlagEvaluations[0] + if event.Flag.Key != "test-flag" { + t.Errorf("unexpected flag key: got %q, want test-flag", event.Flag.Key) + } + if event.EvaluationCount != 1 { + t.Errorf("unexpected evaluation count: got %d, want 1", event.EvaluationCount) + } + case <-time.After(2 * time.Second): + t.Fatal("stop() returned without flushing queued flagevaluation event") + } +} + func TestRecordQueuesPrunedContextSnapshot(t *testing.T) { w := newFlagEvaluationWriter(ProviderConfig{}) attrs := make(map[string]any, maxContextFields+50) diff --git a/openfeature/provider.go b/openfeature/provider.go index 30bf8ba7378..f3e4e5a3345 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -249,7 +249,6 @@ func (p *DatadogProvider) ShutdownWithContext(ctx context.Context) error { } // Stop the EVP flag evaluation writer (nil when killswitch disabled). if p.flagEvalWriter != nil { - p.flagEvalWriter.flush() p.flagEvalWriter.stop() } // Shut down flag evaluation metrics From 57232d82471b935fa806ff9ed6d544ea9ad2e01f Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 18 Jun 2026 21:45:12 -0400 Subject: [PATCH 30/37] fix(openfeature): use singular flagevaluation EVP endpoint --- openfeature/flagevaluation.go | 2 +- openfeature/flagevaluation_test.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 8d96df0d02a..64a36f095a7 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -30,7 +30,7 @@ const ( defaultFlagEvalFlushInterval = 10 * time.Second // flagEvaluationEndpoint is the EVP proxy endpoint for flag evaluation events. - flagEvaluationEndpoint = "/evp_proxy/v2/api/v2/flagevaluations" + flagEvaluationEndpoint = "/evp_proxy/v2/api/v2/flagevaluation" // Context pruning limits — mirror worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH. maxContextFields = 256 diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 1ed10603754..f5390398279 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -34,6 +34,13 @@ func mustMarshalJSONMap(t *testing.T, payload any) map[string]any { return m } +func TestFlagEvaluationEndpointUsesTrackName(t *testing.T) { + const want = "/evp_proxy/v2/api/v2/flagevaluation" + if flagEvaluationEndpoint != want { + t.Fatalf("flagEvaluationEndpoint = %q, want %q", flagEvaluationEndpoint, want) + } +} + // TestFlattenAndPruneContextEquivalence verifies the merged single-pass // flattenAndPruneContext must produce a pruned result byte-for-byte identical to the prior // two-step flattenContext + pruneContext pipeline across nested, oversized, and >256-field From a59755638d583dabd80ec20d80f160cf5fa6fdf7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 18 Jun 2026 22:14:33 -0400 Subject: [PATCH 31/37] chore(openfeature): document flag evaluation cap sizing --- openfeature/flagevaluation.go | 69 ++++++++++++++---------- openfeature/flagevaluation_scale_test.go | 12 ++--- openfeature/flagevaluation_test.go | 31 +++++++---- 3 files changed, 68 insertions(+), 44 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 64a36f095a7..9e000009290 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -36,18 +36,32 @@ const ( maxContextFields = 256 maxFieldLength = 256 + // Aggregation cap sizing inputs. + evalScaleTargetFlags = 2_500 + evalScaleFullBucketsPerFlag = 50 + evalScaleUsersPerFlag = 1_000 + evalScalePerFlagHeadroomMultiplier = 10 + evalScaleDegradedBucketsPerFlag = 10 + evalScaleFullBucketTarget = evalScaleTargetFlags * evalScaleFullBucketsPerFlag + evalScalePerFlagBucketTarget = evalScalePerFlagHeadroomMultiplier * evalScaleUsersPerFlag + evalScaleDegradedBucketTarget = evalScaleTargetFlags * evalScaleDegradedBucketsPerFlag + // Aggregation caps. // - // The cascade is full → degraded → drop(counted). With no ultra-degraded backstop, - // degradedCap must hold the legitimate degraded cardinality at the team's >=2,500-flag - // scale target, or legitimate counts would be dropped. Degraded cardinality is based only - // on schema-visible dimensions retained by the degraded payload; OpenFeature reason is not - // an EVP field and is not part of bucket identity. globalCap (full-tier) is raised to - // 131,072 so a realistic 2,500-flag × multi-context workload keeps full-fidelity buckets - // before degrading rather than dropping them. + // The aggregation path is full → degraded → drop(counted). degradedCap must hold the + // legitimate degraded cardinality at the target scale, or legitimate counts would be + // dropped. Degraded cardinality is based only on schema-visible dimensions retained by + // the degraded payload; OpenFeature reason is not an EVP field and is not part of bucket + // identity. + // + // Sizing arithmetic: + // - globalCap 131,072 (2^17) > evalScaleFullBucketTarget = 125,000 + // - perFlagCap 10,000 = evalScalePerFlagBucketTarget + // - degradedCap 32,768 (2^15) > evalScaleDegradedBucketTarget = 25,000 + // These are starting caps, not schema limits; overflow is counted and dropped. defaultEvalGlobalCap = 131_072 // bounds full-tier buckets only; degraded is bounded separately - defaultEvalPerFlagCap = 10_000 // bounds full-fidelity buckets per flag - defaultEvalDegradedCap = 32_768 // bounds degraded map; overflow is dropped(counted), no ultra tier + defaultEvalPerFlagCap = evalScalePerFlagBucketTarget + defaultEvalDegradedCap = 32_768 // bounds degraded buckets; overflow is counted and dropped // defaultEvalEventBufferSize bounds the async hand-off queue between the (hot-path) // Finally hook and the background aggregation worker. On overflow the hook drops the @@ -82,11 +96,10 @@ type evaluationAggregationKey struct { contextKey string // exact canonical encoding of the pruned context; comparable, not a digest } -// evaluationDegradedKey is the key for the degraded aggregation map — the terminal aggregation -// tier in the 2-tier design (full → degraded → drop). Drops targeting key, context, and -// targeting rule key relative to the full key. It keeps only schema-visible fields emitted by -// the degraded payload. When a NEW degraded bucket would exceed degradedCap, the count is -// dropped and counted (aggregator.dropped) rather than cascading to a further-degraded tier. +// evaluationDegradedKey is the key for the degraded aggregation map in the +// full → degraded → drop path. It drops targeting key, context, and other full-only fields, +// keeping only schema-visible fields emitted by the degraded payload. When a NEW degraded +// bucket would exceed degradedCap, the count is dropped and counted. type evaluationDegradedKey struct { flagKey string variant string @@ -142,9 +155,9 @@ type flagEvaluationAggregator struct { perFlagCap int degradedCap int // dropped counts evaluations whose count was lost because a NEW degraded bucket would have - // exceeded degradedCap (the terminal tier in the 2-tier design). It is the observable signal - // that legitimate counts are being dropped and that degradedCap should be raised. It is - // distinct from flagEvaluationWriter.dropped (which counts async-queue backpressure drops). + // exceeded degradedCap. It is the observable signal that legitimate counts are being + // dropped and that degradedCap should be raised. It is distinct from flagEvaluationWriter.dropped + // (which counts async-queue backpressure drops). droppedDegradedOverflow int64 } @@ -357,8 +370,8 @@ func (w *flagEvaluationWriter) flush() { full := w.aggregator.full degraded := w.aggregator.degraded - // Surface degraded-overflow drops (the terminal-tier backstop in the 2-tier design) so an - // undersized degradedCap is observable rather than a silent loss of legitimate counts. + // Surface degraded-overflow drops so an undersized degradedCap is observable rather than + // a silent loss of legitimate counts. degradedOverflow := w.aggregator.droppedDegradedOverflow if (len(full) + len(degraded)) == 0 { @@ -591,12 +604,12 @@ func (w *flagEvaluationWriter) sendToAgent(payload flagEvaluationPayload) error // add records one evaluation observation into the appropriate aggregation tier. // Must be called WITHOUT the aggregator lock held (it acquires the lock internally). -// Implements the two-tier cascade: full → degraded → drop(counted). +// Routes observations through full → degraded → drop(counted). // // Per-flag attempt counting: perFlagFull[flag] is incremented on every call for a flag // (whether or not a full-tier bucket is actually created). This ensures that once // globalCap is full, a flag that accumulates enough attempts (>= perFlagCap) still -// overflows to degraded — keeping the per-flag overflow path alive even after the +// follows the degraded path — keeping the per-flag overflow path alive even after the // global full-tier cap is exhausted. func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]any, nowMs int64) { a.mu.Lock() @@ -638,7 +651,7 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an if a.globalCount >= a.globalCap { // Global full-tier cap full — count must not be lost. Route into the degraded tier // (which drops targeting_key + context), sized to hold the legitimate degraded - // cardinality at the >=2,500-flag scale target. The per-flag attempt counter was + // cardinality at the target scale. The per-flag attempt counter was // already incremented above; once it reaches perFlagCap this flag routes through // addToDegraded directly as well. a.addToDegraded(d, nowMs) @@ -659,13 +672,11 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an } // addToDegraded adds an entry to the degraded map (drops targeting_key + context). -// Called with the aggregator lock held. Degraded is the TERMINAL aggregation tier in the -// 2-tier design: when a NEW degraded bucket would exceed degradedCap, the evaluation's count is -// DROPPED and counted (droppedDegradedOverflow) rather than cascading to a further-degraded -// tier. degradedCap is sized (defaultEvalDegradedCap) to hold the legitimate degraded -// cardinality at the >=2,500-flag scale target, so this drop only fires under cardinality far -// beyond that target (e.g. an unbounded dynamic/abusive flag key) and the -// dropped counter makes such overflow observable. +// Called with the aggregator lock held. When a NEW degraded bucket would exceed degradedCap, +// the evaluation's count is DROPPED and counted (droppedDegradedOverflow). degradedCap is +// sized (defaultEvalDegradedCap) to hold the legitimate degraded cardinality at the target +// scale, so this drop only fires under cardinality far beyond that target (e.g. an unbounded +// dynamic/abusive flag key) and the dropped counter makes such overflow observable. func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { degKey := evaluationDegradedKey{ flagKey: d.flagKey, diff --git a/openfeature/flagevaluation_scale_test.go b/openfeature/flagevaluation_scale_test.go index 07a15d95242..0333b34ac6b 100644 --- a/openfeature/flagevaluation_scale_test.go +++ b/openfeature/flagevaluation_scale_test.go @@ -44,7 +44,7 @@ func makeScaleFlags(n int) []scaleFlagShape { // legitimateDegradedCardinality returns the number of DISTINCT degraded buckets the given flag // shapes would produce if every (variant × allocation) combination were observed. -// This is the count a 2-tier design must be able to hold without dropping. +// This is the count degradedCap must hold without dropping legitimate buckets. func legitimateDegradedCardinality(flags []scaleFlagShape) int { total := 0 for _, f := range flags { @@ -60,7 +60,7 @@ func legitimateDegradedCardinality(flags []scaleFlagShape) int { // // The context cardinality knob (numContexts) is what splits the FULL tier: the full key includes // targetingKey + contextKey, so more distinct subjects => more full buckets => earlier full-tier -// saturation => earlier cascade into degraded (and potentially ultra). +// saturation => earlier cascade into degraded. func driveScale(agg *flagEvaluationAggregator, flags []scaleFlagShape, numContexts, evalsPerCombo int) int64 { nowMs := time.Now().UnixMilli() var calls int64 @@ -94,8 +94,8 @@ func driveScale(agg *flagEvaluationAggregator, flags []scaleFlagShape, numContex return calls } -// tierCounts returns observable aggregator state after a run. In the 2-tier design the terminal -// tier is degraded; over-cap counts land in droppedDegradedOverflow (observable, not silent). +// tierCounts returns observable aggregator state after a run. Over-cap degraded counts land in +// droppedDegradedOverflow (observable, not silent). // sumCounts includes the drop counter so the count-preservation invariant is Σ == add() calls. type tierCounts struct { full, degraded int @@ -173,7 +173,7 @@ func TestScaleDropTriggerSweep(t *testing.T) { flags := makeScaleFlags(n) deg := legitimateDegradedCardinality(flags) t.Logf("2,500 flags; legitimate degraded cardinality = %d; production caps "+ - "full=%d perFlag=%d degraded=%d (2-tier: full -> degraded -> drop)", + "full=%d perFlag=%d degraded=%d (full -> degraded -> drop)", deg, defaultEvalGlobalCap, defaultEvalPerFlagCap, defaultEvalDegradedCap) // Sweep distinct-context cardinality. Low = few subjects (full tier stays small); @@ -238,7 +238,7 @@ func TestScaleDropRequiresDegradedSaturation(t *testing.T) { tc.full, tc.degraded, tc.dropped, tc.sumCounts, calls) if deg > defaultEvalDegradedCap { - t.Errorf("legitimate degraded cardinality %d EXCEEDS degradedCap %d — 2-tier design would DROP "+ + t.Errorf("legitimate degraded cardinality %d EXCEEDS degradedCap %d — aggregation would DROP "+ "legitimate counts at 2,500 flags; raise degradedCap", deg, defaultEvalDegradedCap) } else { t.Logf(" => all %d legitimate degraded buckets FIT under degradedCap %d (headroom %d); "+ diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index f5390398279..f3fab60b359 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -41,6 +41,20 @@ func TestFlagEvaluationEndpointUsesTrackName(t *testing.T) { } } +func TestDefaultEvalCapSizing(t *testing.T) { + if defaultEvalGlobalCap <= evalScaleFullBucketTarget { + t.Fatalf("defaultEvalGlobalCap = %d, want > %d", defaultEvalGlobalCap, evalScaleFullBucketTarget) + } + + if defaultEvalPerFlagCap != evalScalePerFlagBucketTarget { + t.Fatalf("defaultEvalPerFlagCap = %d, want %d", defaultEvalPerFlagCap, evalScalePerFlagBucketTarget) + } + + if defaultEvalDegradedCap <= evalScaleDegradedBucketTarget { + t.Fatalf("defaultEvalDegradedCap = %d, want > %d", defaultEvalDegradedCap, evalScaleDegradedBucketTarget) + } +} + // TestFlattenAndPruneContextEquivalence verifies the merged single-pass // flattenAndPruneContext must produce a pruned result byte-for-byte identical to the prior // two-step flattenContext + pruneContext pipeline across nested, oversized, and >256-field @@ -611,9 +625,8 @@ func TestAggregatorConcurrentMinMax(t *testing.T) { } } -// TestSaturationCountPreservation is the regression guard against a SILENT drop at saturation -// in the 2-tier design. Because the degraded tier is now terminal (overflow is dropped, not -// cascaded), the invariant is: Σ(full+degraded counts) + droppedDegradedOverflow == add() calls. +// TestSaturationCountPreservation is the regression guard against a SILENT drop at saturation. +// The invariant is: Σ(full+degraded counts) + droppedDegradedOverflow == add() calls. // No evaluation may vanish without being COUNTED — silent loss is the defect this guards against. func TestSaturationCountPreservation(t *testing.T) { // Use small caps so we can saturate them quickly. @@ -723,8 +736,8 @@ func TestAggregatorCapOverflow(t *testing.T) { } } - // Continue adding until degradedCap is also exhausted. At that point — with no ultra - // tier — new degraded buckets must be dropped and COUNTED (droppedDegradedOverflow). + // Continue adding until degradedCap is also exhausted. At that point, new degraded + // buckets must be dropped and COUNTED (droppedDegradedOverflow). for i := range 10 { d := evalDetails{ flagKey: fmt.Sprintf("overflow-flag-%d", i), @@ -980,10 +993,10 @@ func TestCanonicalContextKeyEncoding(t *testing.T) { }) } -// TestDegradedCapBounded verifies that unbounded dynamic/abusive flag keys stay bounded under -// the 2-tier design. With the degraded tier as the terminal tier, an unbounded number of distinct -// flag keys must NOT grow the degraded map without bound: len(degraded) <= degradedCap, and the -// over-cap counts must be DROPPED-AND-COUNTED (droppedDegradedOverflow), never silently lost. +// TestDegradedCapBounded verifies that unbounded dynamic/abusive flag keys stay bounded. +// An unbounded number of distinct flag keys must NOT grow the degraded map without bound: +// len(degraded) <= degradedCap, and the over-cap counts must be DROPPED-AND-COUNTED +// (droppedDegradedOverflow), never silently lost. // Σ(full+degraded counts) + droppedDegradedOverflow must equal the add() call count. func TestDegradedCapBounded(t *testing.T) { const cap = 3 From 218c40919fa38e8e1a24a2f158072a3eb1164653 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 18 Jun 2026 22:15:28 -0400 Subject: [PATCH 32/37] test(openfeature): cover out-of-order evaluation timestamps --- openfeature/flagevaluation.go | 4 +++- openfeature/flagevaluation_test.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 9e000009290..45814a570d7 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -122,7 +122,9 @@ type evaluationEntry struct { // observe records one more evaluation against an existing bucket: it bumps the count and // widens the [firstEvaluation, lastEvaluation] window to include nowMs. Every existing-bucket -// path across the three tiers funnels through here so the count++/min/max logic lives once. +// path across the aggregation tiers funnels through here so the count++/min/max logic lives once. +// nowMs is captured before enqueue; concurrent evaluations can reach the worker out of timestamp +// order, so an existing bucket can legitimately observe an older timestamp. func (e *evaluationEntry) observe(nowMs int64) { e.count++ if nowMs < e.firstEvaluation { diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index f3fab60b359..95a21487e3f 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -625,6 +625,23 @@ func TestAggregatorConcurrentMinMax(t *testing.T) { } } +func TestEvaluationEntryObserveOutOfOrderTimestamps(t *testing.T) { + entry := newEvaluationEntry(200) + + entry.observe(150) + entry.observe(250) + + if entry.count != 3 { + t.Fatalf("count = %d, want 3", entry.count) + } + if entry.firstEvaluation != 150 { + t.Fatalf("firstEvaluation = %d, want 150", entry.firstEvaluation) + } + if entry.lastEvaluation != 250 { + t.Fatalf("lastEvaluation = %d, want 250", entry.lastEvaluation) + } +} + // TestSaturationCountPreservation is the regression guard against a SILENT drop at saturation. // The invariant is: Σ(full+degraded counts) + droppedDegradedOverflow == add() calls. // No evaluation may vanish without being COUNTED — silent loss is the defect this guards against. From b1d4ea5a50180af0b48fdd2fc9ce59fbf954cb2b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 07:17:27 -0400 Subject: [PATCH 33/37] refactor(openfeature): clarify evaluation timestamp naming --- openfeature/flagevaluation.go | 73 ++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 45814a570d7..f63b8341732 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -121,28 +121,29 @@ type evaluationEntry struct { } // observe records one more evaluation against an existing bucket: it bumps the count and -// widens the [firstEvaluation, lastEvaluation] window to include nowMs. Every existing-bucket +// widens the [firstEvaluation, lastEvaluation] window to include evaluationTimeMs. Every existing-bucket // path across the aggregation tiers funnels through here so the count++/min/max logic lives once. -// nowMs is captured before enqueue; concurrent evaluations can reach the worker out of timestamp -// order, so an existing bucket can legitimately observe an older timestamp. -func (e *evaluationEntry) observe(nowMs int64) { +// evaluationTimeMs is captured before enqueue; concurrent evaluations can reach the worker out +// of timestamp order, so an existing bucket can legitimately observe an older timestamp. +func (e *evaluationEntry) observe(evaluationTimeMs int64) { e.count++ - if nowMs < e.firstEvaluation { - e.firstEvaluation = nowMs + if evaluationTimeMs < e.firstEvaluation { + e.firstEvaluation = evaluationTimeMs } - if nowMs > e.lastEvaluation { - e.lastEvaluation = nowMs + if evaluationTimeMs > e.lastEvaluation { + e.lastEvaluation = evaluationTimeMs } } -// newEvaluationEntry returns a fresh bucket for nowMs with count 1 and first==last==nowMs. +// newEvaluationEntry returns a fresh bucket for evaluationTimeMs with count 1 and +// first==last==evaluationTimeMs. // Callers set any tier-specific fields (runtimeDefault, targetingKey, contextAttrs, // errorMessage) on the returned entry. -func newEvaluationEntry(nowMs int64) *evaluationEntry { +func newEvaluationEntry(evaluationTimeMs int64) *evaluationEntry { return &evaluationEntry{ count: 1, - firstEvaluation: nowMs, - lastEvaluation: nowMs, + firstEvaluation: evaluationTimeMs, + lastEvaluation: evaluationTimeMs, } } @@ -248,9 +249,9 @@ type flagEvaluationWriter struct { // evalEvent is the bounded snapshot the Finally hook hands to the worker. contextAttrs is already // flattened and pruned, so the async queue never buffers the caller's raw evaluation context. type evalEvent struct { - d evalDetails - contextAttrs map[string]any - nowMs int64 + d evalDetails + contextAttrs map[string]any + evaluationTimeMs int64 } // evalDetails holds extracted flag evaluation fields for EVP aggregation. @@ -398,13 +399,13 @@ func (w *flagEvaluationWriter) flush() { log.Warn("openfeature: degraded aggregation tier full — dropped %d evaluation(s); raise degradedCap (best-effort telemetry)", degradedOverflow) } - nowMs := time.Now().UnixMilli() + flushTimeMs := time.Now().UnixMilli() var events []flagEvaluationEvent // Full tier: required fields + variant + allocation + targeting_key + context + error. // runtime_default_used decorates this tier when the caller default was returned. for key, e := range full { - ev := baseFlagEvaluationEvent(key.flagKey, e, nowMs) + ev := baseFlagEvaluationEvent(key.flagKey, e, flushTimeMs) ev.RuntimeDefault = e.runtimeDefault ev.TargetingKey = e.targetingKey if key.variant != "" { @@ -425,7 +426,7 @@ func (w *flagEvaluationWriter) flush() { // Degraded tier: required fields + variant + allocation + error; no targeting_key, no context. // runtime_default_used decorates this tier when the caller default was returned. for key, e := range degraded { - ev := baseFlagEvaluationEvent(key.flagKey, e, nowMs) + ev := baseFlagEvaluationEvent(key.flagKey, e, flushTimeMs) ev.RuntimeDefault = e.runtimeDefault if key.variant != "" { ev.Variant = &flagEvalVariant{Key: key.variant} @@ -459,9 +460,9 @@ func (w *flagEvaluationWriter) flush() { // fields (timestamp, flag.key, first/last evaluation, evaluation_count). It is tier-agnostic // and sets no optional field — RuntimeDefault and the rest are decoration applied by each tier // loop in flush() after the call. -func baseFlagEvaluationEvent(flagKey string, e *evaluationEntry, nowMs int64) flagEvaluationEvent { +func baseFlagEvaluationEvent(flagKey string, e *evaluationEntry, flushTimeMs int64) flagEvaluationEvent { return flagEvaluationEvent{ - Timestamp: nowMs, + Timestamp: flushTimeMs, Flag: flagEvalFlag{Key: flagKey}, FirstEvaluation: e.firstEvaluation, LastEvaluation: e.lastEvaluation, @@ -489,14 +490,14 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int // Use the evaluation time captured by the provider (most-correct; see metadataEvalTimeKey). // Fall back to the hook-fire time only when absent (e.g. a non-Datadog provider that did not // stamp it), so the first/last_evaluation bounds are always populated. - nowMs := d.evalTimeMs - if nowMs == 0 { - nowMs = time.Now().UnixMilli() + evaluationTimeMs := d.evalTimeMs + if evaluationTimeMs == 0 { + evaluationTimeMs = time.Now().UnixMilli() } ev := evalEvent{ - d: d, - contextAttrs: flattenAndPruneContext(hookContext.EvaluationContext().Attributes()), - nowMs: nowMs, + d: d, + contextAttrs: flattenAndPruneContext(hookContext.EvaluationContext().Attributes()), + evaluationTimeMs: evaluationTimeMs, } select { case w.events <- ev: @@ -507,7 +508,7 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int // aggregate updates the aggregator. It runs only on the writer's single worker goroutine. func (w *flagEvaluationWriter) aggregate(ev evalEvent) { - w.aggregator.add(ev.d, ev.contextAttrs, ev.nowMs) + w.aggregator.add(ev.d, ev.contextAttrs, ev.evaluationTimeMs) } // flattenAndPruneContext produces the pruned context map for EVP aggregation in a single @@ -613,7 +614,7 @@ func (w *flagEvaluationWriter) sendToAgent(payload flagEvaluationPayload) error // globalCap is full, a flag that accumulates enough attempts (>= perFlagCap) still // follows the degraded path — keeping the per-flag overflow path alive even after the // global full-tier cap is exhausted. -func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]any, nowMs int64) { +func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]any, evaluationTimeMs int64) { a.mu.Lock() defer a.mu.Unlock() @@ -633,14 +634,14 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an // contextKey is the full canonical encoding (not a digest), this fast path is hit only by a // genuinely identical pruned context — never by an aliasing collision. if e, ok := a.full[fullKey]; ok { - e.observe(nowMs) + e.observe(evaluationTimeMs) return } // Check per-flag cap. if a.perFlagFull[d.flagKey] >= a.perFlagCap { // perFlagCap exceeded — route to degraded tier. - a.addToDegraded(d, nowMs) + a.addToDegraded(d, evaluationTimeMs) return } @@ -656,15 +657,15 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an // cardinality at the target scale. The per-flag attempt counter was // already incremented above; once it reaches perFlagCap this flag routes through // addToDegraded directly as well. - a.addToDegraded(d, nowMs) + a.addToDegraded(d, evaluationTimeMs) return } // New full-tier entry. a.full[fullKey] = &evaluationEntry{ count: 1, - firstEvaluation: nowMs, - lastEvaluation: nowMs, + firstEvaluation: evaluationTimeMs, + lastEvaluation: evaluationTimeMs, runtimeDefault: d.runtimeDefault, targetingKey: d.targetingKey, contextAttrs: contextAttrs, @@ -679,7 +680,7 @@ func (a *flagEvaluationAggregator) add(d evalDetails, contextAttrs map[string]an // sized (defaultEvalDegradedCap) to hold the legitimate degraded cardinality at the target // scale, so this drop only fires under cardinality far beyond that target (e.g. an unbounded // dynamic/abusive flag key) and the dropped counter makes such overflow observable. -func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { +func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, evaluationTimeMs int64) { degKey := evaluationDegradedKey{ flagKey: d.flagKey, variant: d.variant, @@ -689,7 +690,7 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { } if e, ok := a.degraded[degKey]; ok { - e.observe(nowMs) + e.observe(evaluationTimeMs) return } @@ -701,7 +702,7 @@ func (a *flagEvaluationAggregator) addToDegraded(d evalDetails, nowMs int64) { return } - e := newEvaluationEntry(nowMs) + e := newEvaluationEntry(evaluationTimeMs) e.runtimeDefault = d.runtimeDefault e.errorMessage = d.errorMessage a.degraded[degKey] = e From d1c04c8e6472b1b27136816328b8f32221e12f8b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 21:39:02 -0400 Subject: [PATCH 34/37] Limit OpenFeature benchmark gate to baseline-backed cases --- .gitlab/benchmarks/micro/gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/benchmarks/micro/gitlab-ci.yml b/.gitlab/benchmarks/micro/gitlab-ci.yml index 31d336450e3..36b82f5dadd 100644 --- a/.gitlab/benchmarks/micro/gitlab-ci.yml +++ b/.gitlab/benchmarks/micro/gitlab-ci.yml @@ -86,7 +86,7 @@ microbenchmarks-2: microbenchmarks-3: extends: .microbenchmarks variables: - BENCHMARKS: "BenchmarkOpenFeatureClientEvaluation|BenchmarkOpenFeatureClientConcurrentEvaluations|BenchmarkBooleanEvaluation|BenchmarkStringEvaluation|BenchmarkIntEvaluation|BenchmarkFloatEvaluation|BenchmarkEvaluation|BenchmarkEvaluationWithVaryingContextSize|BenchmarkEvaluationWithVaryingFlagCounts|BenchmarkConcurrentEvaluations|BenchmarkFlagEvaluationNoop|BenchmarkFlagEvaluationOTelOnly|BenchmarkFlagEvaluationOTelPlusEVP|BenchmarkFlagEvaluationEVPRecord|BenchmarkFlagEvaluationOTelPlusEVPParallel" + BENCHMARKS: "BenchmarkOpenFeatureClientEvaluation|BenchmarkOpenFeatureClientConcurrentEvaluations|BenchmarkBooleanEvaluation|BenchmarkStringEvaluation|BenchmarkIntEvaluation|BenchmarkFloatEvaluation|BenchmarkEvaluation|BenchmarkEvaluationWithVaryingContextSize|BenchmarkEvaluationWithVaryingFlagCounts|BenchmarkConcurrentEvaluations" CPUS_PER_BENCHMARK: 1 REPETITIONS: 10 GROUP_ID: "group-3" From 3fb4e80e6b16cd46b8b123f748e4a1ab35a0a770 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 22:18:30 -0400 Subject: [PATCH 35/37] fix(openfeature): bound flagevaluation enqueue work --- openfeature/evaluator.go | 2 +- openfeature/flagevaluation.go | 26 ++++++++++++++++++++++++- openfeature/flagevaluation_hook_test.go | 25 ++++++++++++++++++++++++ openfeature/flagevaluation_test.go | 15 ++++++++++++++ openfeature/provider.go | 10 ++-------- openfeature/provider_test.go | 3 +-- 6 files changed, 69 insertions(+), 12 deletions(-) diff --git a/openfeature/evaluator.go b/openfeature/evaluator.go index b9c38150430..dbf66ad74d5 100644 --- a/openfeature/evaluator.go +++ b/openfeature/evaluator.go @@ -82,7 +82,7 @@ func evaluateFlag(flag *flag, defaultValue any, context map[string]any, now time } // Build metadata for exposure tracking - metadata := make(map[string]any) + metadata := make(map[string]any, 2) metadata[metadataAllocationKey] = allocation.Key // Get doLog value (defaults to true if not specified) diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index f63b8341732..1a4bf6da69a 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -364,7 +364,7 @@ func (w *flagEvaluationWriter) stop() { func (w *flagEvaluationWriter) flush() { // Surface best-effort backpressure drops (queue full) as an observable signal. if d := w.dropped.Swap(0); d > 0 { - log.Warn("openfeature: flag evaluation queue full — dropped %d evaluation(s) under backpressure (best-effort telemetry)", d) + log.Debug("openfeature: flag evaluation queue full — dropped %d evaluation(s) under backpressure (best-effort telemetry)", d) } w.aggregator.mu.Lock() @@ -486,6 +486,10 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int w.dropped.Add(1) return } + if len(w.events) == cap(w.events) { + w.dropped.Add(1) + return + } d := extractEvalDetails(hookContext, details) // Use the evaluation time captured by the provider (most-correct; see metadataEvalTimeKey). // Fall back to the hook-fire time only when absent (e.g. a non-Datadog provider that did not @@ -532,6 +536,26 @@ func flattenAndPruneContext(attrs map[string]any) map[string]any { if len(attrs) == 0 { return nil } + if len(attrs) <= maxContextFields { + needsFlattenOrPrune := false + for _, v := range attrs { + switch x := v.(type) { + case string: + if len(x) > maxFieldLength { + needsFlattenOrPrune = true + } + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, bool: + default: + needsFlattenOrPrune = true + } + if needsFlattenOrPrune { + break + } + } + if !needsFlattenOrPrune { + return attrs + } + } flat := make(map[string]any, len(attrs)) flattenRecursive("", attrs, flat) diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index 48367488977..d2c844e0d14 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -62,6 +62,12 @@ func makeEvalDetails(variant string, reason of.Reason, errorCode of.ErrorCode, m return d } +type panicStringer struct{} + +func (panicStringer) String() string { + panic("String must not be called when the queue is already full") +} + // TestIsRuntimeDefault verifies the runtime-default detection rule. // Signal: absent variant key. Our evaluator sets a variant ONLY on a matched // allocation (TARGETING_MATCH/SPLIT/STATIC); every DEFAULT/DISABLED/ERROR path @@ -327,3 +333,22 @@ func TestFlagEvaluationBackpressureDrops(t *testing.T) { t.Errorf("expected exactly %d dropped evaluations once the queue filled, got %d", overflow, got) } } + +func TestFlagEvaluationBackpressureDropsBeforeContextSnapshot(t *testing.T) { + w := setupTestWriter(t) + for range cap(w.events) { + w.events <- evalEvent{} + } + + hookCtx := makeHookContext("bp-flag", "user-1", map[string]any{"expensive": panicStringer{}}) + details := makeEvalDetails("on", of.TargetingMatchReason, "") + + w.record(hookCtx, details) + + if got := w.dropped.Load(); got != 1 { + t.Fatalf("expected one dropped evaluation when queue is already full, got %d", got) + } + if got := len(w.events); got != cap(w.events) { + t.Fatalf("full queue length changed: got %d, want %d", got, cap(w.events)) + } +} diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 95a21487e3f..676616fce27 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -128,6 +128,21 @@ func TestFlattenAndPruneContextEquivalence(t *testing.T) { } } +func TestFlattenAndPruneContextFlatFastPath(t *testing.T) { + attrs := map[string]any{ + "country": "US", + "age": 42, + "premium": true, + } + + got := flattenAndPruneContext(attrs) + got["observed-fast-path"] = "same-map" + + if attrs["observed-fast-path"] != "same-map" { + t.Fatal("flat scalar context should use the already-snapshotted attributes map") + } +} + // setupTestAggregator creates a flagEvaluationAggregator with small caps for testing. // Caps are deliberately small to trigger tier-cascade behavior in unit tests. func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { diff --git a/openfeature/provider.go b/openfeature/provider.go index f3e4e5a3345..6fddfd3313d 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -470,17 +470,11 @@ func (p *DatadogProvider) evaluate( defaultValue any, flatCtx openfeature.FlattenedContext, ) (res evaluationResult) { - // Capture the evaluation time once, at evaluation entry — the most-correct "eval time". It is - // reused for the allocation time-window checks (passed into evaluateFlag) and stamped into the - // result metadata below, so the EVP flagevaluation hook records eval-time instead of the later - // hook-fire time, with no extra time.Now() on the eval path. + // Capture the evaluation time once, at evaluation entry. It is reused for allocation + // time-window checks and stamped into result metadata so EVP first/last bounds use eval-time. evalNow := time.Now() log.Debug("openfeature: evaluating flag %q", flagKey) defer func() { - // Stamp eval-time into metadata on EVERY return path (matched, default, disabled, error, - // ctx-cancelled, no-config, not-found). On the matched path the metadata map already exists - // (allocation key + doLog), so this only adds a key; on other paths it allocates a 1-entry - // map. Read back by extractEvalDetails for the EVP first/last_evaluation bounds. if res.Metadata == nil { res.Metadata = make(map[string]any, 1) } diff --git a/openfeature/provider_test.go b/openfeature/provider_test.go index 52f537988d2..eb5f6ae640f 100644 --- a/openfeature/provider_test.go +++ b/openfeature/provider_test.go @@ -263,8 +263,7 @@ func TestNewDatadogProvider(t *testing.T) { } // TestEvaluateStampsEvalTimeMetadata verifies the provider stamps the evaluation time into -// FlagMetadata (metadataEvalTimeKey) on EVERY path — matched, default, disabled, not-found — so -// the EVP flagevaluation hook records eval-time rather than the later hook-fire time. +// FlagMetadata on every path so EVP first/last bounds use eval-time. func TestEvaluateStampsEvalTimeMetadata(t *testing.T) { provider := newDatadogProvider(ProviderConfig{}) provider.updateConfiguration(createTestConfig()) From 7b43059023201cd6afc3766e4a81e0e084dca8d5 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 23:26:16 -0400 Subject: [PATCH 36/37] fix(openfeature): reduce flagevaluation hot-path work --- openfeature/exposure_hook.go | 5 -- openfeature/flagevaluation.go | 79 +++++++++++-------------- openfeature/flagevaluation_hook_test.go | 71 ++++------------------ openfeature/flagevaluation_test.go | 33 ++++------- openfeature/provider.go | 13 +--- openfeature/provider_test.go | 18 ++---- 6 files changed, 66 insertions(+), 153 deletions(-) diff --git a/openfeature/exposure_hook.go b/openfeature/exposure_hook.go index c1a47879212..301f72a7437 100644 --- a/openfeature/exposure_hook.go +++ b/openfeature/exposure_hook.go @@ -19,11 +19,6 @@ const ( // Metadata keys for exposure tracking metadataAllocationKey = "dd.allocation.key" metadataDoLogKey = "dd.doLog" - // metadataEvalTimeKey carries the evaluation timestamp (UnixMilli, int64). It is stamped in - // DatadogProvider.evaluate at evaluation entry — the most-correct evaluation time — so the EVP - // flagevaluation hook records eval-time rather than the later, slightly-incorrect hook-fire - // time. Read back in extractEvalDetails (flagevaluation.go). - metadataEvalTimeKey = "dd.eval.timestamp_ms" ) // exposureHook implements the OpenFeature Hook interface to track feature flag exposures. diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index 1a4bf6da69a..e033081b1ee 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -246,12 +246,10 @@ type flagEvaluationWriter struct { enqueueMu sync.RWMutex } -// evalEvent is the bounded snapshot the Finally hook hands to the worker. contextAttrs is already -// flattened and pruned, so the async queue never buffers the caller's raw evaluation context. +// evalEvent is the bounded snapshot the Finally hook hands to the worker. type evalEvent struct { - d evalDetails - contextAttrs map[string]any - evaluationTimeMs int64 + d evalDetails + evaluationContext of.EvaluationContext } // evalDetails holds extracted flag evaluation fields for EVP aggregation. @@ -263,10 +261,6 @@ type evalDetails struct { targetingKey string errorMessage string runtimeDefault bool - // evalTimeMs is the evaluation timestamp (UnixMilli) captured by the provider at eval entry - // and passed through flag metadata. 0 when absent (e.g. a non-Datadog provider), in which case - // record() falls back to the hook-fire time. - evalTimeMs int64 } // newFlagEvaluationWriter creates a new flag evaluation writer. @@ -471,10 +465,10 @@ func baseFlagEvaluationEvent(flagKey string, e *evaluationEntry, flushTimeMs int } // record runs on the evaluation hot path (the Finally hook). It does only cheap scalar -// extraction plus a bounded context snapshot, then a non-blocking enqueue — no aggregation -// happens here; the background worker does that. If the queue is full the event is dropped -// and counted (best-effort), never blocking the evaluation. Called from the Finally hook after -// every evaluation. +// extraction plus an immutable context handoff, then a non-blocking enqueue — no aggregation +// or context flattening happens here; the background worker does that. If the queue is full +// the event is dropped and counted (best-effort), never blocking the evaluation. Called from +// the Finally hook after every evaluation. func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.InterfaceEvaluationDetails) { w.enqueueMu.RLock() defer w.enqueueMu.RUnlock() @@ -491,17 +485,9 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int return } d := extractEvalDetails(hookContext, details) - // Use the evaluation time captured by the provider (most-correct; see metadataEvalTimeKey). - // Fall back to the hook-fire time only when absent (e.g. a non-Datadog provider that did not - // stamp it), so the first/last_evaluation bounds are always populated. - evaluationTimeMs := d.evalTimeMs - if evaluationTimeMs == 0 { - evaluationTimeMs = time.Now().UnixMilli() - } ev := evalEvent{ - d: d, - contextAttrs: flattenAndPruneContext(hookContext.EvaluationContext().Attributes()), - evaluationTimeMs: evaluationTimeMs, + d: d, + evaluationContext: hookContext.EvaluationContext(), } select { case w.events <- ev: @@ -512,7 +498,8 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int // aggregate updates the aggregator. It runs only on the writer's single worker goroutine. func (w *flagEvaluationWriter) aggregate(ev evalEvent) { - w.aggregator.add(ev.d, ev.contextAttrs, ev.evaluationTimeMs) + contextAttrs := flattenAndPruneContext(ev.evaluationContext.Attributes()) + w.aggregator.add(ev.d, contextAttrs, time.Now().UnixMilli()) } // flattenAndPruneContext produces the pruned context map for EVP aggregation in a single @@ -536,25 +523,8 @@ func flattenAndPruneContext(attrs map[string]any) map[string]any { if len(attrs) == 0 { return nil } - if len(attrs) <= maxContextFields { - needsFlattenOrPrune := false - for _, v := range attrs { - switch x := v.(type) { - case string: - if len(x) > maxFieldLength { - needsFlattenOrPrune = true - } - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, bool: - default: - needsFlattenOrPrune = true - } - if needsFlattenOrPrune { - break - } - } - if !needsFlattenOrPrune { - return attrs - } + if contextFitsWithoutFlattening(attrs) { + return attrs } flat := make(map[string]any, len(attrs)) @@ -609,6 +579,26 @@ func flattenAndPruneContext(attrs map[string]any) map[string]any { return out } +func contextFitsWithoutFlattening(attrs map[string]any) bool { + if len(attrs) > maxContextFields { + return false + } + for _, v := range attrs { + switch x := v.(type) { + case string: + if len(x) > maxFieldLength { + return false + } + case int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64, + float32, float64, bool: + default: + return false + } + } + return true +} + // drainAndFlush processes any buffered events and performs a final flush. Called by the // worker when stopping so a final batch is not lost on shutdown. func (w *flagEvaluationWriter) drainAndFlush() { @@ -879,8 +869,6 @@ func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluati if errMsg == "" && details.ErrorCode != "" { errMsg = string(details.ErrorCode) } - // Evaluation time, stamped by DatadogProvider.evaluate at eval entry. 0 when absent. - evalTimeMs, _ := details.FlagMetadata[metadataEvalTimeKey].(int64) return evalDetails{ flagKey: hookContext.FlagKey(), variant: details.Variant, @@ -888,6 +876,5 @@ func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluati targetingKey: hookContext.EvaluationContext().TargetingKey(), errorMessage: errMsg, runtimeDefault: isRuntimeDefault(details), - evalTimeMs: evalTimeMs, } } diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index d2c844e0d14..7004c75d382 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -145,67 +145,20 @@ func TestIsRuntimeDefault(t *testing.T) { } } -// TestExtractEvalDetailsReadsEvalTime verifies the provider-stamped evaluation time -// (metadataEvalTimeKey) is read into evalDetails.evalTimeMs, and is 0 when absent. -func TestExtractEvalDetailsReadsEvalTime(t *testing.T) { - const evalTime int64 = 1_717_171_717_123 - tests := []struct { - name string - md of.FlagMetadata - want int64 - }{ - {"eval-time present", of.FlagMetadata{metadataEvalTimeKey: evalTime}, evalTime}, - {"eval-time present alongside allocation key", of.FlagMetadata{metadataAllocationKey: "alloc-1", metadataEvalTimeKey: evalTime}, evalTime}, - {"eval-time absent", of.FlagMetadata{metadataAllocationKey: "alloc-1"}, 0}, - {"nil metadata", nil, 0}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - hookCtx := makeHookContext("flag-x", "user-1", nil) - details := makeEvalDetails("on", of.TargetingMatchReason, "", tc.md) - if got := extractEvalDetails(hookCtx, details).evalTimeMs; got != tc.want { - t.Errorf("evalTimeMs = %d, want %d", got, tc.want) - } - }) - } -} +// TestRecordUsesAggregationTime verifies aggregated entries receive a worker-side timestamp. +func TestRecordUsesAggregationTime(t *testing.T) { + before := time.Now().UnixMilli() + w := setupTestWriter(t) + w.record(makeHookContext("flag-z", "user-3", nil), + makeEvalDetails("on", of.TargetingMatchReason, "")) + w.aggregate(<-w.events) + after := time.Now().UnixMilli() -// TestRecordUsesEvalTimeFromMetadata verifies record() stamps the aggregated entry's -// first/last evaluation from the provider-supplied eval-time (metadataEvalTimeKey), and -// falls back to the hook-fire time only when that metadata is absent. Fails on the prior -// behavior, which always used time.Now() in record(). -func TestRecordUsesEvalTimeFromMetadata(t *testing.T) { - t.Run("uses provider eval-time", func(t *testing.T) { - const evalTime int64 = 1_700_000_000_000 // a fixed time in the past, distinct from now - w := setupTestWriter(t) - w.record(makeHookContext("flag-y", "user-2", nil), - makeEvalDetails("on", of.TargetingMatchReason, "", of.FlagMetadata{metadataEvalTimeKey: evalTime})) - w.aggregate(<-w.events) - - if len(w.aggregator.full) != 1 { - t.Fatalf("expected 1 full-tier entry, got %d", len(w.aggregator.full)) - } - for _, e := range w.aggregator.full { - if e.firstEvaluation != evalTime || e.lastEvaluation != evalTime { - t.Errorf("entry first/last = %d/%d, want %d (provider eval-time)", e.firstEvaluation, e.lastEvaluation, evalTime) - } - } - }) - - t.Run("falls back to hook-fire time when absent", func(t *testing.T) { - before := time.Now().UnixMilli() - w := setupTestWriter(t) - w.record(makeHookContext("flag-z", "user-3", nil), - makeEvalDetails("on", of.TargetingMatchReason, "")) // no eval-time metadata - w.aggregate(<-w.events) - after := time.Now().UnixMilli() - - for _, e := range w.aggregator.full { - if e.firstEvaluation < before || e.firstEvaluation > after { - t.Errorf("fallback first=%d not within hook-fire window [%d,%d]", e.firstEvaluation, before, after) - } + for _, e := range w.aggregator.full { + if e.firstEvaluation < before || e.firstEvaluation > after { + t.Errorf("first=%d not within aggregation window [%d,%d]", e.firstEvaluation, before, after) } - }) + } } // TestFlagEvaluationHookFinally verifies that the Finally hook records an entry for diff --git a/openfeature/flagevaluation_test.go b/openfeature/flagevaluation_test.go index 676616fce27..ac64f2319da 100644 --- a/openfeature/flagevaluation_test.go +++ b/openfeature/flagevaluation_test.go @@ -128,21 +128,6 @@ func TestFlattenAndPruneContextEquivalence(t *testing.T) { } } -func TestFlattenAndPruneContextFlatFastPath(t *testing.T) { - attrs := map[string]any{ - "country": "US", - "age": 42, - "premium": true, - } - - got := flattenAndPruneContext(attrs) - got["observed-fast-path"] = "same-map" - - if attrs["observed-fast-path"] != "same-map" { - t.Fatal("flat scalar context should use the already-snapshotted attributes map") - } -} - // setupTestAggregator creates a flagEvaluationAggregator with small caps for testing. // Caps are deliberately small to trigger tier-cascade behavior in unit tests. func setupTestAggregator(t *testing.T) *flagEvaluationAggregator { @@ -1185,7 +1170,7 @@ func TestStopDrainsAndFlushesQueuedFlagEvaluations(t *testing.T) { } } -func TestRecordQueuesPrunedContextSnapshot(t *testing.T) { +func TestRecordAggregatesPrunedContextSnapshot(t *testing.T) { w := newFlagEvaluationWriter(ProviderConfig{}) attrs := make(map[string]any, maxContextFields+50) for i := range maxContextFields + 50 { @@ -1217,12 +1202,18 @@ func TestRecordQueuesPrunedContextSnapshot(t *testing.T) { if len(w.events) != 1 { t.Fatalf("expected one queued event, got %d", len(w.events)) } - ev := <-w.events - if got := len(ev.contextAttrs); got != maxContextFields { - t.Fatalf("queued context should be pruned to %d fields, got %d", maxContextFields, got) + w.aggregate(<-w.events) + + if len(w.aggregator.full) != 1 { + t.Fatalf("expected one aggregated entry, got %d", len(w.aggregator.full)) } - if _, ok := ev.contextAttrs["zzz-oversized"]; ok { - t.Fatal("queued context should not contain oversized string values") + for _, entry := range w.aggregator.full { + if got := len(entry.contextAttrs); got != maxContextFields { + t.Fatalf("aggregated context should be pruned to %d fields, got %d", maxContextFields, got) + } + if _, ok := entry.contextAttrs["zzz-oversized"]; ok { + t.Fatal("aggregated context should not contain oversized string values") + } } } diff --git a/openfeature/provider.go b/openfeature/provider.go index 6fddfd3313d..626df041f5a 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -469,18 +469,11 @@ func (p *DatadogProvider) evaluate( flagKey string, defaultValue any, flatCtx openfeature.FlattenedContext, -) (res evaluationResult) { - // Capture the evaluation time once, at evaluation entry. It is reused for allocation - // time-window checks and stamped into result metadata so EVP first/last bounds use eval-time. +) evaluationResult { + // Capture the evaluation time once, at evaluation entry. It is used for allocation + // time-window checks. evalNow := time.Now() log.Debug("openfeature: evaluating flag %q", flagKey) - defer func() { - if res.Metadata == nil { - res.Metadata = make(map[string]any, 1) - } - res.Metadata[metadataEvalTimeKey] = evalNow.UnixMilli() - log.Debug("openfeature: evaluated flag %q: value=%v, reason=%s, error=%v", flagKey, res.Value, res.Reason, res.Error) - }() // Check if context was cancelled before starting evaluation select { diff --git a/openfeature/provider_test.go b/openfeature/provider_test.go index eb5f6ae640f..20ab7c6761c 100644 --- a/openfeature/provider_test.go +++ b/openfeature/provider_test.go @@ -262,9 +262,10 @@ func TestNewDatadogProvider(t *testing.T) { } } -// TestEvaluateStampsEvalTimeMetadata verifies the provider stamps the evaluation time into -// FlagMetadata on every path so EVP first/last bounds use eval-time. -func TestEvaluateStampsEvalTimeMetadata(t *testing.T) { +// TestEvaluateDoesNotStampEvalTimeMetadata verifies the provider does not add EVP-only +// timestamp metadata to normal evaluation results. +func TestEvaluateDoesNotStampEvalTimeMetadata(t *testing.T) { + const metadataEvalTimeKey = "dd.eval.timestamp_ms" provider := newDatadogProvider(ProviderConfig{}) provider.updateConfiguration(createTestConfig()) ctx := context.Background() @@ -282,18 +283,11 @@ func TestEvaluateStampsEvalTimeMetadata(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - before := time.Now().UnixMilli() result := provider.BooleanEvaluation(ctx, tc.flagKey, false, tc.flatCtx) - after := time.Now().UnixMilli() - - ts, ok := result.FlagMetadata[metadataEvalTimeKey].(int64) - if !ok { - t.Fatalf("FlagMetadata[%q] missing or not int64 on %q path; metadata=%v", + if _, ok := result.FlagMetadata[metadataEvalTimeKey]; ok { + t.Fatalf("FlagMetadata[%q] should not be stamped on %q path; metadata=%v", metadataEvalTimeKey, tc.name, result.FlagMetadata) } - if ts < before || ts > after { - t.Errorf("eval-time %d not within evaluation window [%d,%d]", ts, before, after) - } }) } } From f2a6df08644a15aecaead0f68ca1e646dac276b3 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 23:49:47 -0400 Subject: [PATCH 37/37] fix(openfeature): keep flagevaluation benchmarks informational --- .gitlab/benchmarks/micro/gitlab-ci.yml | 2 +- openfeature/evaluator.go | 2 +- openfeature/exposure_hook.go | 3 ++ openfeature/flagevaluation.go | 11 +++- openfeature/flagevaluation_hook_test.go | 69 ++++++++++++++++++++----- openfeature/provider.go | 10 +++- openfeature/provider_test.go | 19 ++++--- 7 files changed, 93 insertions(+), 23 deletions(-) diff --git a/.gitlab/benchmarks/micro/gitlab-ci.yml b/.gitlab/benchmarks/micro/gitlab-ci.yml index 36b82f5dadd..886a1178c3d 100644 --- a/.gitlab/benchmarks/micro/gitlab-ci.yml +++ b/.gitlab/benchmarks/micro/gitlab-ci.yml @@ -93,7 +93,7 @@ microbenchmarks-3: pr-performance-gates: stage: microbenchmarks - needs: [microbenchmarks-1, microbenchmarks-2, microbenchmarks-3] + needs: [microbenchmarks-1, microbenchmarks-2] rules: *benchmark-rules tags: - "arch:amd64" diff --git a/openfeature/evaluator.go b/openfeature/evaluator.go index dbf66ad74d5..58021cc3f82 100644 --- a/openfeature/evaluator.go +++ b/openfeature/evaluator.go @@ -82,7 +82,7 @@ func evaluateFlag(flag *flag, defaultValue any, context map[string]any, now time } // Build metadata for exposure tracking - metadata := make(map[string]any, 2) + metadata := make(map[string]any, 3) metadata[metadataAllocationKey] = allocation.Key // Get doLog value (defaults to true if not specified) diff --git a/openfeature/exposure_hook.go b/openfeature/exposure_hook.go index 301f72a7437..31ad834e39a 100644 --- a/openfeature/exposure_hook.go +++ b/openfeature/exposure_hook.go @@ -19,6 +19,9 @@ const ( // Metadata keys for exposure tracking metadataAllocationKey = "dd.allocation.key" metadataDoLogKey = "dd.doLog" + // metadataEvalTimeKey carries the evaluation timestamp (UnixMilli, int64). It is stamped in + // DatadogProvider.evaluate at evaluation entry so EVP first/last bounds use eval-time. + metadataEvalTimeKey = "dd.eval.timestamp_ms" ) // exposureHook implements the OpenFeature Hook interface to track feature flag exposures. diff --git a/openfeature/flagevaluation.go b/openfeature/flagevaluation.go index e033081b1ee..6575ae9c96f 100644 --- a/openfeature/flagevaluation.go +++ b/openfeature/flagevaluation.go @@ -250,6 +250,7 @@ type flagEvaluationWriter struct { type evalEvent struct { d evalDetails evaluationContext of.EvaluationContext + evaluationTimeMs int64 } // evalDetails holds extracted flag evaluation fields for EVP aggregation. @@ -261,6 +262,7 @@ type evalDetails struct { targetingKey string errorMessage string runtimeDefault bool + evalTimeMs int64 } // newFlagEvaluationWriter creates a new flag evaluation writer. @@ -485,9 +487,14 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int return } d := extractEvalDetails(hookContext, details) + evaluationTimeMs := d.evalTimeMs + if evaluationTimeMs == 0 { + evaluationTimeMs = time.Now().UnixMilli() + } ev := evalEvent{ d: d, evaluationContext: hookContext.EvaluationContext(), + evaluationTimeMs: evaluationTimeMs, } select { case w.events <- ev: @@ -499,7 +506,7 @@ func (w *flagEvaluationWriter) record(hookContext of.HookContext, details of.Int // aggregate updates the aggregator. It runs only on the writer's single worker goroutine. func (w *flagEvaluationWriter) aggregate(ev evalEvent) { contextAttrs := flattenAndPruneContext(ev.evaluationContext.Attributes()) - w.aggregator.add(ev.d, contextAttrs, time.Now().UnixMilli()) + w.aggregator.add(ev.d, contextAttrs, ev.evaluationTimeMs) } // flattenAndPruneContext produces the pruned context map for EVP aggregation in a single @@ -869,6 +876,7 @@ func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluati if errMsg == "" && details.ErrorCode != "" { errMsg = string(details.ErrorCode) } + evalTimeMs, _ := details.FlagMetadata[metadataEvalTimeKey].(int64) return evalDetails{ flagKey: hookContext.FlagKey(), variant: details.Variant, @@ -876,5 +884,6 @@ func extractEvalDetails(hookContext of.HookContext, details of.InterfaceEvaluati targetingKey: hookContext.EvaluationContext().TargetingKey(), errorMessage: errMsg, runtimeDefault: isRuntimeDefault(details), + evalTimeMs: evalTimeMs, } } diff --git a/openfeature/flagevaluation_hook_test.go b/openfeature/flagevaluation_hook_test.go index 7004c75d382..4fb6979f2fa 100644 --- a/openfeature/flagevaluation_hook_test.go +++ b/openfeature/flagevaluation_hook_test.go @@ -145,20 +145,65 @@ func TestIsRuntimeDefault(t *testing.T) { } } -// TestRecordUsesAggregationTime verifies aggregated entries receive a worker-side timestamp. -func TestRecordUsesAggregationTime(t *testing.T) { - before := time.Now().UnixMilli() - w := setupTestWriter(t) - w.record(makeHookContext("flag-z", "user-3", nil), - makeEvalDetails("on", of.TargetingMatchReason, "")) - w.aggregate(<-w.events) - after := time.Now().UnixMilli() +// TestExtractEvalDetailsReadsEvalTime verifies provider-stamped evaluation time +// is read into evalDetails.evalTimeMs, and is 0 when absent. +func TestExtractEvalDetailsReadsEvalTime(t *testing.T) { + const evalTime int64 = 1_717_171_717_123 + tests := []struct { + name string + md of.FlagMetadata + want int64 + }{ + {"eval-time present", of.FlagMetadata{metadataEvalTimeKey: evalTime}, evalTime}, + {"eval-time present alongside allocation key", of.FlagMetadata{metadataAllocationKey: "alloc-1", metadataEvalTimeKey: evalTime}, evalTime}, + {"eval-time absent", of.FlagMetadata{metadataAllocationKey: "alloc-1"}, 0}, + {"nil metadata", nil, 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + hookCtx := makeHookContext("flag-x", "user-1", nil) + details := makeEvalDetails("on", of.TargetingMatchReason, "", tc.md) + if got := extractEvalDetails(hookCtx, details).evalTimeMs; got != tc.want { + t.Errorf("evalTimeMs = %d, want %d", got, tc.want) + } + }) + } +} - for _, e := range w.aggregator.full { - if e.firstEvaluation < before || e.firstEvaluation > after { - t.Errorf("first=%d not within aggregation window [%d,%d]", e.firstEvaluation, before, after) +// TestRecordUsesEvalTimeFromMetadata verifies record() stamps the aggregated entry's +// first/last evaluation from provider-supplied eval-time, and falls back to hook time when absent. +func TestRecordUsesEvalTimeFromMetadata(t *testing.T) { + t.Run("uses provider eval-time", func(t *testing.T) { + const evalTime int64 = 1_700_000_000_000 + w := setupTestWriter(t) + w.record(makeHookContext("flag-y", "user-2", nil), + makeEvalDetails("on", of.TargetingMatchReason, "", of.FlagMetadata{metadataEvalTimeKey: evalTime})) + w.aggregate(<-w.events) + + if len(w.aggregator.full) != 1 { + t.Fatalf("expected 1 full-tier entry, got %d", len(w.aggregator.full)) } - } + for _, e := range w.aggregator.full { + if e.firstEvaluation != evalTime || e.lastEvaluation != evalTime { + t.Errorf("entry first/last = %d/%d, want %d", e.firstEvaluation, e.lastEvaluation, evalTime) + } + } + }) + + t.Run("falls back to hook time when absent", func(t *testing.T) { + before := time.Now().UnixMilli() + w := setupTestWriter(t) + w.record(makeHookContext("flag-z", "user-3", nil), + makeEvalDetails("on", of.TargetingMatchReason, "")) + w.aggregate(<-w.events) + after := time.Now().UnixMilli() + + for _, e := range w.aggregator.full { + if e.firstEvaluation < before || e.firstEvaluation > after { + t.Errorf("fallback first=%d not within hook window [%d,%d]", e.firstEvaluation, before, after) + } + } + }) } // TestFlagEvaluationHookFinally verifies that the Finally hook records an entry for diff --git a/openfeature/provider.go b/openfeature/provider.go index 626df041f5a..d385a9d1a07 100644 --- a/openfeature/provider.go +++ b/openfeature/provider.go @@ -469,11 +469,17 @@ func (p *DatadogProvider) evaluate( flagKey string, defaultValue any, flatCtx openfeature.FlattenedContext, -) evaluationResult { +) (res evaluationResult) { // Capture the evaluation time once, at evaluation entry. It is used for allocation - // time-window checks. + // time-window checks and EVP first/last evaluation bounds. evalNow := time.Now() log.Debug("openfeature: evaluating flag %q", flagKey) + defer func() { + if res.Metadata == nil { + res.Metadata = make(map[string]any, 1) + } + res.Metadata[metadataEvalTimeKey] = evalNow.UnixMilli() + }() // Check if context was cancelled before starting evaluation select { diff --git a/openfeature/provider_test.go b/openfeature/provider_test.go index 20ab7c6761c..51e6d6489c7 100644 --- a/openfeature/provider_test.go +++ b/openfeature/provider_test.go @@ -262,10 +262,9 @@ func TestNewDatadogProvider(t *testing.T) { } } -// TestEvaluateDoesNotStampEvalTimeMetadata verifies the provider does not add EVP-only -// timestamp metadata to normal evaluation results. -func TestEvaluateDoesNotStampEvalTimeMetadata(t *testing.T) { - const metadataEvalTimeKey = "dd.eval.timestamp_ms" +// TestEvaluateStampsEvalTimeMetadata verifies the provider stamps evaluation-entry +// time into FlagMetadata on every path so EVP first/last bounds use eval-time. +func TestEvaluateStampsEvalTimeMetadata(t *testing.T) { provider := newDatadogProvider(ProviderConfig{}) provider.updateConfiguration(createTestConfig()) ctx := context.Background() @@ -283,11 +282,19 @@ func TestEvaluateDoesNotStampEvalTimeMetadata(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { + before := time.Now().UnixMilli() result := provider.BooleanEvaluation(ctx, tc.flagKey, false, tc.flatCtx) - if _, ok := result.FlagMetadata[metadataEvalTimeKey]; ok { - t.Fatalf("FlagMetadata[%q] should not be stamped on %q path; metadata=%v", + after := time.Now().UnixMilli() + + got, ok := result.FlagMetadata[metadataEvalTimeKey].(int64) + if !ok { + t.Fatalf("FlagMetadata[%q] missing or wrong type on %q path; metadata=%v", metadataEvalTimeKey, tc.name, result.FlagMetadata) } + if got < before || got > after { + t.Fatalf("FlagMetadata[%q] = %d, want within [%d,%d] on %q path", + metadataEvalTimeKey, got, before, after, tc.name) + } }) } }