feat(openfeature): implement flag evaluation metrics (#7993)

dd-oleksii · rochdev · commit cc9745d0f169 · 2026-04-29T06:02:50.000Z
Add an OpenFeature `finally` hook that emits a `feature_flag.evaluations`
OTel counter for every flag evaluation, including error and short-circuit
cases. The hook is a no-op when `otelMetricsEnabled` is false.
diff --git a/packages/dd-trace/src/openfeature/eval-metrics-hook.js b/packages/dd-trace/src/openfeature/eval-metrics-hook.js
@@ -0,0 +1,103 @@
+'use strict'
+
+const log = require('../log')
+
+const METER_NAME = 'dd-trace-js/openfeature'
+const COUNTER_NAME = 'feature_flag.evaluations'
+const COUNTER_DESCRIPTION = 'Number of feature flag evaluations'
+const COUNTER_UNIT = '{evaluation}'
+
+/**
+ * OpenFeature hook that tracks feature flag evaluation metrics using an
+ * OpenTelemetry counter.
+ *
+ * Implements the OpenFeature `finally` hook interface so it can be pushed
+ * directly onto a provider's `hooks` array. We use the `finally` stage
+ * (not diagnostic channels inside the provider's `resolve*` methods) because
+ * the OpenFeature SDK short-circuits before calling the provider when it is in
+ * NOT_READY state; the `finally` hook still fires, ensuring all evaluations are
+ * captured. It also catches type-mismatch errors detected by the SDK client
+ * after the provider returns.
+ *
+ * The counter is created lazily on the first successful `finally()` call rather
+ * than in the constructor. This is necessary because `FlaggingProvider` is
+ * constructed eagerly by `proxy.js#updateTracing()`, which runs *before*
+ * `initializeOpenTelemetryMetrics()` sets the global OTel meter provider.
+ * Calling `getMeter()` in the constructor would return the noop meter and
+ * produce a noop counter that silently discards all measurements. By deferring
+ * to `finally()` time we give the meter provider a chance to be set up first.
+ *
+ * If counter creation fails (e.g. the OTel API is not yet available), the call
+ * is silently skipped and retried on the next `finally()` invocation.
+ *
+ * When `config.otelMetricsEnabled` is false, `finally()` is always a no-op.
+ */
+class EvalMetricsHook {
+  #enabled = false
+  #counter = null
+
+  /**
+   * @param {import('../config')} config - Tracer configuration object
+   */
+  constructor (config) {
+    this.#enabled = config.otelMetricsEnabled === true
+  }
+
+  /**
+   * Returns the OTel counter, creating it on first successful call.
+   * Returns `null` if counter creation fails; will retry on next call.
+   *
+   * @returns {import('@opentelemetry/api').Counter | null}
+   */
+  #getCounter () {
+    if (this.#counter) return this.#counter
+
+    try {
+      const { metrics } = require('@opentelemetry/api')
+      const meter = metrics.getMeter(METER_NAME)
+      this.#counter = meter.createCounter(COUNTER_NAME, {
+        description: COUNTER_DESCRIPTION,
+        unit: COUNTER_UNIT,
+      })
+    } catch (e) {
+      log.warn('EvalMetricsHook: failed to create counter: %s', e.message)
+    }
+
+    return this.#counter
+  }
+
+  /**
+   * Called by the OpenFeature SDK after every flag evaluation (success or error).
+   *
+   * @param {{ flagKey: string }} hookContext - Hook context containing the flag key
+   * @param {{ variant?: string, reason?: string, errorCode?: string, flagMetadata?: object }} evaluationDetails
+   *   - Full evaluation details
+   * @returns {void}
+   */
+  finally (hookContext, evaluationDetails) {
+    if (!this.#enabled) return
+
+    const counter = this.#getCounter()
+    if (!counter) return
+
+    const attributes = {
+      'feature_flag.key': hookContext?.flagKey ?? '',
+      'feature_flag.result.variant': evaluationDetails?.variant ?? '',
+      'feature_flag.result.reason': evaluationDetails?.reason?.toLowerCase() ?? 'unknown',
+    }
+
+    const errorCode = evaluationDetails?.errorCode
+    if (errorCode) {
+      attributes['error.type'] = errorCode.toLowerCase()
+    }
+
+    const allocationKey = evaluationDetails?.flagMetadata?.allocationKey
+    if (allocationKey) {
+      attributes['feature_flag.result.allocation_key'] = allocationKey
+    }
+
+    counter.add(1, attributes)
+  }
+}
+
+module.exports = EvalMetricsHook
diff --git a/packages/dd-trace/src/openfeature/flagging_provider.js b/packages/dd-trace/src/openfeature/flagging_provider.js
@@ -4,6 +4,7 @@ const { DatadogNodeServerProvider } = require('@datadog/openfeature-node-server'
 const { channel } = require('dc-polyfill')
 const log = require('../log')
 const { EXPOSURE_CHANNEL } = require('./constants/constants')
+const EvalMetricsHook = require('./eval-metrics-hook')
 
 /**
  * OpenFeature provider that integrates with Datadog's feature flagging system.
@@ -24,6 +25,8 @@ class FlaggingProvider extends DatadogNodeServerProvider {
     this._tracer = tracer
     this._config = config
 
+    this.hooks.push(new EvalMetricsHook(config))
+
     log.debug('%s created with timeout: %dms', this.constructor.name,
       config.experimental.flaggingProvider.initializationTimeoutMs)
   }
diff --git a/packages/dd-trace/test/openfeature/eval-metrics-hook.spec.js b/packages/dd-trace/test/openfeature/eval-metrics-hook.spec.js
@@ -0,0 +1,213 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+
+const { describe, it, beforeEach } = require('mocha')
+const sinon = require('sinon')
+const proxyquire = require('proxyquire')
+
+require('../setup/core')
+
+describe('EvalMetricsHook', () => {
+  let EvalMetricsHook
+  let mockCounter
+  let mockMeter
+  let mockOtelApi
+  let log
+
+  beforeEach(() => {
+    mockCounter = {
+      add: sinon.spy(),
+    }
+
+    mockMeter = {
+      createCounter: sinon.stub().returns(mockCounter),
+    }
+
+    mockOtelApi = {
+      metrics: {
+        getMeter: sinon.stub().returns(mockMeter),
+      },
+    }
+
+    log = {
+      warn: sinon.spy(),
+      debug: sinon.spy(),
+    }
+
+    EvalMetricsHook = proxyquire('../../src/openfeature/eval-metrics-hook', {
+      '@opentelemetry/api': mockOtelApi,
+      '../log': log,
+    })
+  })
+
+  function makeConfig (otelMetricsEnabled = true) {
+    return { otelMetricsEnabled }
+  }
+
+  function hookContext (flagKey = 'flag') {
+    return { flagKey }
+  }
+
+  function evalDetails (overrides = {}) {
+    return { variant: 'on', reason: 'TARGETING_MATCH', ...overrides }
+  }
+
+  describe('finally()', () => {
+    it('should be a no-op when disabled', () => {
+      const metrics = new EvalMetricsHook(makeConfig(false))
+      metrics.finally(hookContext(), evalDetails())
+
+      sinon.assert.notCalled(mockCounter.add)
+      sinon.assert.notCalled(mockOtelApi.metrics.getMeter)
+    })
+
+    it('should create counter lazily on first call', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      sinon.assert.notCalled(mockOtelApi.metrics.getMeter)
+
+      metrics.finally(hookContext('my-flag'), evalDetails())
+
+      sinon.assert.calledOnceWithExactly(mockOtelApi.metrics.getMeter, 'dd-trace-js/openfeature')
+      sinon.assert.calledWith(mockMeter.createCounter, 'feature_flag.evaluations', {
+        description: 'Number of feature flag evaluations',
+        unit: '{evaluation}',
+      })
+    })
+
+    it('should cache the counter after first call', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext(), evalDetails())
+      metrics.finally(hookContext(), evalDetails())
+
+      sinon.assert.calledOnce(mockOtelApi.metrics.getMeter)
+    })
+
+    it('should add counter with basic attributes', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext('my-flag'), evalDetails())
+
+      sinon.assert.calledOnce(mockCounter.add)
+      const [value, attributes] = mockCounter.add.firstCall.args
+      assert.strictEqual(value, 1)
+      assert.deepStrictEqual(attributes, {
+        'feature_flag.key': 'my-flag',
+        'feature_flag.result.variant': 'on',
+        'feature_flag.result.reason': 'targeting_match',
+      })
+    })
+
+    it('should lowercase the reason', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext(), evalDetails({ reason: 'DEFAULT' }))
+
+      const [, attributes] = mockCounter.add.firstCall.args
+      assert.strictEqual(attributes['feature_flag.result.reason'], 'default')
+    })
+
+    it('should use empty string for variant when variant is undefined', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext(), evalDetails({ variant: undefined }))
+
+      const [, attributes] = mockCounter.add.firstCall.args
+      assert.strictEqual(attributes['feature_flag.result.variant'], '')
+    })
+
+    it('should include error.type when errorCode is set', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(
+        hookContext('flag'),
+        evalDetails({ variant: undefined, reason: 'ERROR', errorCode: 'TYPE_MISMATCH' })
+      )
+
+      const [, attributes] = mockCounter.add.firstCall.args
+      assert.deepStrictEqual(attributes, {
+        'feature_flag.key': 'flag',
+        'feature_flag.result.variant': '',
+        'feature_flag.result.reason': 'error',
+        'error.type': 'type_mismatch',
+      })
+    })
+
+    it('should lowercase errorCode in error.type', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext(), evalDetails({ reason: 'ERROR', errorCode: 'FLAG_NOT_FOUND' }))
+
+      const [, attributes] = mockCounter.add.firstCall.args
+      assert.strictEqual(attributes['error.type'], 'flag_not_found')
+    })
+
+    it('should omit error.type when errorCode is falsy', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext(), evalDetails())
+
+      const [, attributes] = mockCounter.add.firstCall.args
+      assert.ok(!Object.hasOwn(attributes, 'error.type'))
+    })
+
+    it('should include allocation_key when set', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(
+        hookContext('flag'),
+        evalDetails({ flagMetadata: { allocationKey: 'default-allocation' } })
+      )
+
+      const [, attributes] = mockCounter.add.firstCall.args
+      assert.deepStrictEqual(attributes, {
+        'feature_flag.key': 'flag',
+        'feature_flag.result.variant': 'on',
+        'feature_flag.result.reason': 'targeting_match',
+        'feature_flag.result.allocation_key': 'default-allocation',
+      })
+    })
+
+    it('should omit allocation_key when flagMetadata is absent', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext(), evalDetails())
+
+      const [, attributes] = mockCounter.add.firstCall.args
+      assert.ok(!Object.hasOwn(attributes, 'feature_flag.result.allocation_key'))
+    })
+
+    it('should omit allocation_key when flagMetadata is empty', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext(), evalDetails({ flagMetadata: {} }))
+
+      const [, attributes] = mockCounter.add.firstCall.args
+      assert.ok(!Object.hasOwn(attributes, 'feature_flag.result.allocation_key'))
+    })
+
+    it('should skip when OTel api throws', () => {
+      mockOtelApi.metrics.getMeter.throws(new Error('OTel not ready'))
+      const metrics = new EvalMetricsHook(makeConfig(true))
+
+      metrics.finally(hookContext(), evalDetails())
+
+      sinon.assert.notCalled(mockCounter.add)
+      sinon.assert.calledOnce(log.warn)
+
+      // Fix the OTel API and verify retry succeeds
+      mockOtelApi.metrics.getMeter.returns(mockMeter)
+      metrics.finally(hookContext(), evalDetails())
+
+      sinon.assert.calledOnce(mockCounter.add)
+    })
+
+    it('should handle null/undefined hookContext gracefully', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(undefined, evalDetails())
+      metrics.finally(null, evalDetails())
+
+      assert.strictEqual(mockCounter.add.callCount, 2)
+      assert.strictEqual(mockCounter.add.firstCall.args[1]['feature_flag.key'], '')
+    })
+
+    it('should handle null/undefined evaluationDetails gracefully', () => {
+      const metrics = new EvalMetricsHook(makeConfig(true))
+      metrics.finally(hookContext(), undefined)
+      metrics.finally(hookContext(), null)
+
+      assert.strictEqual(mockCounter.add.callCount, 2)
+    })
+  })
+})
diff --git a/packages/dd-trace/test/openfeature/flagging_provider.spec.js b/packages/dd-trace/test/openfeature/flagging_provider.spec.js
@@ -15,6 +15,8 @@ describe('FlaggingProvider', () => {
   let mockChannel
   let log
   let channelStub
+  let mockEvalMetricsHook
+  let mockEvalMetricsHookClass
 
   beforeEach(() => {
     mockTracer = {
@@ -45,11 +47,17 @@ describe('FlaggingProvider', () => {
       warn: sinon.spy(),
     }
 
+    mockEvalMetricsHook = {
+      record: sinon.spy(),
+    }
+    mockEvalMetricsHookClass = sinon.stub().returns(mockEvalMetricsHook)
+
     FlaggingProvider = proxyquire('../../src/openfeature/flagging_provider', {
       'dc-polyfill': {
         channel: channelStub,
       },
       '../log': log,
+      './eval-metrics-hook': mockEvalMetricsHookClass,
     })
   })
 
@@ -96,6 +104,21 @@ describe('FlaggingProvider', () => {
     })
   })
 
+  describe('hooks', () => {
+    it('should create EvalMetricsHook with config', () => {
+      new FlaggingProvider(mockTracer, mockConfig) // eslint-disable-line no-new
+
+      sinon.assert.calledOnceWithExactly(mockEvalMetricsHookClass, mockConfig)
+    })
+
+    it('should register EvalMetricsHook as a hook', () => {
+      const provider = new FlaggingProvider(mockTracer, mockConfig)
+
+      assert.strictEqual(provider.hooks.length, 1)
+      assert.strictEqual(provider.hooks[0], mockEvalMetricsHook)
+    })
+  })
+
   describe('inheritance', () => {
     it('should extend DatadogNodeServerProvider', () => {
       const { DatadogNodeServerProvider } = require('@datadog/openfeature-node-server')
