Skip to content

Commit 43bb33c

Browse files
dd-oleksiidd-oleksii
committed
feat(openfeature): implement flag evaluation metrics
Add an OpenFeature `finally` hook that emits a `feature_flag.evaluations` OTel counter for every flag evaluation, including error and short-circuit cases. The hook is a no-op when `otelMetricsEnabled` is false.
1 parent df1f326 commit 43bb33c

4 files changed

Lines changed: 342 additions & 0 deletions

File tree

‎packages/dd-trace/src/openfeature/eval-metrics-hook.js‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
'use strict'
2+
3+
const log = require('../log')
4+
5+
const METER_NAME = 'dd-trace-js/openfeature'
6+
const COUNTER_NAME = 'feature_flag.evaluations'
7+
const COUNTER_DESCRIPTION = 'Number of feature flag evaluations'
8+
const COUNTER_UNIT = '{evaluation}'
9+
10+
/**
11+
* OpenFeature hook that tracks feature flag evaluation metrics using an
12+
* OpenTelemetry counter.
13+
*
14+
* Implements the OpenFeature `finally` hook interface so it can be pushed
15+
* directly onto a provider's `hooks` array. We use the `finally` stage
16+
* (not diagnostic channels inside the provider's `resolve*` methods) because
17+
* the OpenFeature SDK short-circuits before calling the provider when it is in
18+
* NOT_READY state; the `finally` hook still fires, ensuring all evaluations are
19+
* captured. It also catches type-mismatch errors detected by the SDK client
20+
* after the provider returns.
21+
*
22+
* The counter is created lazily on the first successful `finally()` call rather
23+
* than in the constructor. This is necessary because `FlaggingProvider` is
24+
* constructed eagerly by `proxy.js#updateTracing()`, which runs *before*
25+
* `initializeOpenTelemetryMetrics()` sets the global OTel meter provider.
26+
* Calling `getMeter()` in the constructor would return the noop meter and
27+
* produce a noop counter that silently discards all measurements. By deferring
28+
* to `finally()` time we give the meter provider a chance to be set up first.
29+
*
30+
* If counter creation fails (e.g. the OTel API is not yet available), the call
31+
* is silently skipped and retried on the next `finally()` invocation.
32+
*
33+
* When `config.otelMetricsEnabled` is false, `finally()` is always a no-op.
34+
*/
35+
class EvalMetricsHook {
36+
#enabled = false
37+
#counter = null
38+
39+
/**
40+
* @param {import('../config')} config - Tracer configuration object
41+
*/
42+
constructor (config) {
43+
this.#enabled = config.otelMetricsEnabled === true
44+
}
45+
46+
/**
47+
* Returns the OTel counter, creating it on first successful call.
48+
* Returns `null` if counter creation fails; will retry on next call.
49+
*
50+
* @returns {import('@opentelemetry/api').Counter | null}
51+
*/
52+
#getCounter () {
53+
if (this.#counter) return this.#counter
54+
55+
try {
56+
const { metrics } = require('@opentelemetry/api')
57+
const meter = metrics.getMeter(METER_NAME)
58+
this.#counter = meter.createCounter(COUNTER_NAME, {
59+
description: COUNTER_DESCRIPTION,
60+
unit: COUNTER_UNIT,
61+
})
62+
} catch (e) {
63+
log.warn('EvalMetricsHook: failed to create counter: %s', e.message)
64+
}
65+
66+
return this.#counter
67+
}
68+
69+
/**
70+
* Called by the OpenFeature SDK after every flag evaluation (success or error).
71+
*
72+
* @param {{ flagKey: string }} hookContext - Hook context containing the flag key
73+
* @param {{ variant?: string, reason?: string, errorCode?: string, flagMetadata?: object }} evaluationDetails
74+
* - Full evaluation details
75+
* @returns {void}
76+
*/
77+
finally (hookContext, evaluationDetails) {
78+
if (!this.#enabled) return
79+
80+
const counter = this.#getCounter()
81+
if (!counter) return
82+
83+
const attributes = {
84+
'feature_flag.key': hookContext?.flagKey ?? '',
85+
'feature_flag.result.variant': evaluationDetails?.variant ?? '',
86+
'feature_flag.result.reason': evaluationDetails?.reason?.toLowerCase() ?? 'unknown',
87+
}
88+
89+
const errorCode = evaluationDetails?.errorCode
90+
if (errorCode) {
91+
attributes['error.type'] = errorCode.toLowerCase()
92+
}
93+
94+
const allocationKey = evaluationDetails?.flagMetadata?.allocationKey
95+
if (allocationKey) {
96+
attributes['feature_flag.result.allocation_key'] = allocationKey
97+
}
98+
99+
counter.add(1, attributes)
100+
}
101+
}
102+
103+
module.exports = EvalMetricsHook

‎packages/dd-trace/src/openfeature/flagging_provider.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const { DatadogNodeServerProvider } = require('@datadog/openfeature-node-server'
44
const { channel } = require('dc-polyfill')
55
const log = require('../log')
66
const { EXPOSURE_CHANNEL } = require('./constants/constants')
7+
const EvalMetricsHook = require('./eval-metrics-hook')
78

89
/**
910
* OpenFeature provider that integrates with Datadog's feature flagging system.
@@ -24,6 +25,8 @@ class FlaggingProvider extends DatadogNodeServerProvider {
2425
this._tracer = tracer
2526
this._config = config
2627

28+
this.hooks.push(new EvalMetricsHook(config))
29+
2730
log.debug('%s created with timeout: %dms', this.constructor.name,
2831
config.experimental.flaggingProvider.initializationTimeoutMs)
2932
}

‎packages/dd-trace/test/openfeature/eval-metrics-hook.spec.js‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
'use strict'
2+
3+
const assert = require('node:assert/strict')
4+
5+
const { describe, it, beforeEach } = require('mocha')
6+
const sinon = require('sinon')
7+
const proxyquire = require('proxyquire')
8+
9+
require('../setup/core')
10+
11+
describe('EvalMetricsHook', () => {
12+
let EvalMetricsHook
13+
let mockCounter
14+
let mockMeter
15+
let mockOtelApi
16+
let log
17+
18+
beforeEach(() => {
19+
mockCounter = {
20+
add: sinon.spy(),
21+
}
22+
23+
mockMeter = {
24+
createCounter: sinon.stub().returns(mockCounter),
25+
}
26+
27+
mockOtelApi = {
28+
metrics: {
29+
getMeter: sinon.stub().returns(mockMeter),
30+
},
31+
}
32+
33+
log = {
34+
warn: sinon.spy(),
35+
debug: sinon.spy(),
36+
}
37+
38+
EvalMetricsHook = proxyquire('../../src/openfeature/eval-metrics-hook', {
39+
'@opentelemetry/api': mockOtelApi,
40+
'../log': log,
41+
})
42+
})
43+
44+
function makeConfig (otelMetricsEnabled = true) {
45+
return { otelMetricsEnabled }
46+
}
47+
48+
function hookContext (flagKey = 'flag') {
49+
return { flagKey }
50+
}
51+
52+
function evalDetails (overrides = {}) {
53+
return { variant: 'on', reason: 'TARGETING_MATCH', ...overrides }
54+
}
55+
56+
describe('finally()', () => {
57+
it('should be a no-op when disabled', () => {
58+
const metrics = new EvalMetricsHook(makeConfig(false))
59+
metrics.finally(hookContext(), evalDetails())
60+
61+
sinon.assert.notCalled(mockCounter.add)
62+
sinon.assert.notCalled(mockOtelApi.metrics.getMeter)
63+
})
64+
65+
it('should create counter lazily on first call', () => {
66+
const metrics = new EvalMetricsHook(makeConfig(true))
67+
sinon.assert.notCalled(mockOtelApi.metrics.getMeter)
68+
69+
metrics.finally(hookContext('my-flag'), evalDetails())
70+
71+
sinon.assert.calledOnceWithExactly(mockOtelApi.metrics.getMeter, 'dd-trace-js/openfeature')
72+
sinon.assert.calledWith(mockMeter.createCounter, 'feature_flag.evaluations', {
73+
description: 'Number of feature flag evaluations',
74+
unit: '{evaluation}',
75+
})
76+
})
77+
78+
it('should cache the counter after first call', () => {
79+
const metrics = new EvalMetricsHook(makeConfig(true))
80+
metrics.finally(hookContext(), evalDetails())
81+
metrics.finally(hookContext(), evalDetails())
82+
83+
sinon.assert.calledOnce(mockOtelApi.metrics.getMeter)
84+
})
85+
86+
it('should add counter with basic attributes', () => {
87+
const metrics = new EvalMetricsHook(makeConfig(true))
88+
metrics.finally(hookContext('my-flag'), evalDetails())
89+
90+
sinon.assert.calledOnce(mockCounter.add)
91+
const [value, attributes] = mockCounter.add.firstCall.args
92+
assert.strictEqual(value, 1)
93+
assert.deepStrictEqual(attributes, {
94+
'feature_flag.key': 'my-flag',
95+
'feature_flag.result.variant': 'on',
96+
'feature_flag.result.reason': 'targeting_match',
97+
})
98+
})
99+
100+
it('should lowercase the reason', () => {
101+
const metrics = new EvalMetricsHook(makeConfig(true))
102+
metrics.finally(hookContext(), evalDetails({ reason: 'DEFAULT' }))
103+
104+
const [, attributes] = mockCounter.add.firstCall.args
105+
assert.strictEqual(attributes['feature_flag.result.reason'], 'default')
106+
})
107+
108+
it('should use empty string for variant when variant is undefined', () => {
109+
const metrics = new EvalMetricsHook(makeConfig(true))
110+
metrics.finally(hookContext(), evalDetails({ variant: undefined }))
111+
112+
const [, attributes] = mockCounter.add.firstCall.args
113+
assert.strictEqual(attributes['feature_flag.result.variant'], '')
114+
})
115+
116+
it('should include error.type when errorCode is set', () => {
117+
const metrics = new EvalMetricsHook(makeConfig(true))
118+
metrics.finally(
119+
hookContext('flag'),
120+
evalDetails({ variant: undefined, reason: 'ERROR', errorCode: 'TYPE_MISMATCH' })
121+
)
122+
123+
const [, attributes] = mockCounter.add.firstCall.args
124+
assert.deepStrictEqual(attributes, {
125+
'feature_flag.key': 'flag',
126+
'feature_flag.result.variant': '',
127+
'feature_flag.result.reason': 'error',
128+
'error.type': 'type_mismatch',
129+
})
130+
})
131+
132+
it('should lowercase errorCode in error.type', () => {
133+
const metrics = new EvalMetricsHook(makeConfig(true))
134+
metrics.finally(hookContext(), evalDetails({ reason: 'ERROR', errorCode: 'FLAG_NOT_FOUND' }))
135+
136+
const [, attributes] = mockCounter.add.firstCall.args
137+
assert.strictEqual(attributes['error.type'], 'flag_not_found')
138+
})
139+
140+
it('should omit error.type when errorCode is falsy', () => {
141+
const metrics = new EvalMetricsHook(makeConfig(true))
142+
metrics.finally(hookContext(), evalDetails())
143+
144+
const [, attributes] = mockCounter.add.firstCall.args
145+
assert.ok(!Object.hasOwn(attributes, 'error.type'))
146+
})
147+
148+
it('should include allocation_key when set', () => {
149+
const metrics = new EvalMetricsHook(makeConfig(true))
150+
metrics.finally(
151+
hookContext('flag'),
152+
evalDetails({ flagMetadata: { allocationKey: 'default-allocation' } })
153+
)
154+
155+
const [, attributes] = mockCounter.add.firstCall.args
156+
assert.deepStrictEqual(attributes, {
157+
'feature_flag.key': 'flag',
158+
'feature_flag.result.variant': 'on',
159+
'feature_flag.result.reason': 'targeting_match',
160+
'feature_flag.result.allocation_key': 'default-allocation',
161+
})
162+
})
163+
164+
it('should omit allocation_key when flagMetadata is absent', () => {
165+
const metrics = new EvalMetricsHook(makeConfig(true))
166+
metrics.finally(hookContext(), evalDetails())
167+
168+
const [, attributes] = mockCounter.add.firstCall.args
169+
assert.ok(!Object.hasOwn(attributes, 'feature_flag.result.allocation_key'))
170+
})
171+
172+
it('should omit allocation_key when flagMetadata is empty', () => {
173+
const metrics = new EvalMetricsHook(makeConfig(true))
174+
metrics.finally(hookContext(), evalDetails({ flagMetadata: {} }))
175+
176+
const [, attributes] = mockCounter.add.firstCall.args
177+
assert.ok(!Object.hasOwn(attributes, 'feature_flag.result.allocation_key'))
178+
})
179+
180+
it('should skip when OTel api throws', () => {
181+
mockOtelApi.metrics.getMeter.throws(new Error('OTel not ready'))
182+
const metrics = new EvalMetricsHook(makeConfig(true))
183+
184+
metrics.finally(hookContext(), evalDetails())
185+
186+
sinon.assert.notCalled(mockCounter.add)
187+
sinon.assert.calledOnce(log.warn)
188+
189+
// Fix the OTel API and verify retry succeeds
190+
mockOtelApi.metrics.getMeter.returns(mockMeter)
191+
metrics.finally(hookContext(), evalDetails())
192+
193+
sinon.assert.calledOnce(mockCounter.add)
194+
})
195+
196+
it('should handle null/undefined hookContext gracefully', () => {
197+
const metrics = new EvalMetricsHook(makeConfig(true))
198+
metrics.finally(undefined, evalDetails())
199+
metrics.finally(null, evalDetails())
200+
201+
assert.strictEqual(mockCounter.add.callCount, 2)
202+
assert.strictEqual(mockCounter.add.firstCall.args[1]['feature_flag.key'], '')
203+
})
204+
205+
it('should handle null/undefined evaluationDetails gracefully', () => {
206+
const metrics = new EvalMetricsHook(makeConfig(true))
207+
metrics.finally(hookContext(), undefined)
208+
metrics.finally(hookContext(), null)
209+
210+
assert.strictEqual(mockCounter.add.callCount, 2)
211+
})
212+
})
213+
})

‎packages/dd-trace/test/openfeature/flagging_provider.spec.js‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ describe('FlaggingProvider', () => {
1515
let mockChannel
1616
let log
1717
let channelStub
18+
let mockEvalMetricsHook
19+
let mockEvalMetricsHookClass
1820

1921
beforeEach(() => {
2022
mockTracer = {
@@ -45,11 +47,17 @@ describe('FlaggingProvider', () => {
4547
warn: sinon.spy(),
4648
}
4749

50+
mockEvalMetricsHook = {
51+
record: sinon.spy(),
52+
}
53+
mockEvalMetricsHookClass = sinon.stub().returns(mockEvalMetricsHook)
54+
4855
FlaggingProvider = proxyquire('../../src/openfeature/flagging_provider', {
4956
'dc-polyfill': {
5057
channel: channelStub,
5158
},
5259
'../log': log,
60+
'./eval-metrics-hook': mockEvalMetricsHookClass,
5361
})
5462
})
5563

@@ -96,6 +104,21 @@ describe('FlaggingProvider', () => {
96104
})
97105
})
98106

107+
describe('hooks', () => {
108+
it('should create EvalMetricsHook with config', () => {
109+
new FlaggingProvider(mockTracer, mockConfig) // eslint-disable-line no-new
110+
111+
sinon.assert.calledOnceWithExactly(mockEvalMetricsHookClass, mockConfig)
112+
})
113+
114+
it('should register EvalMetricsHook as a hook', () => {
115+
const provider = new FlaggingProvider(mockTracer, mockConfig)
116+
117+
assert.strictEqual(provider.hooks.length, 1)
118+
assert.strictEqual(provider.hooks[0], mockEvalMetricsHook)
119+
})
120+
})
121+
99122
describe('inheritance', () => {
100123
it('should extend DatadogNodeServerProvider', () => {
101124
const { DatadogNodeServerProvider } = require('@datadog/openfeature-node-server')

0 commit comments

Comments
 (0)