feat!: Add per-execution runId and at-most-once event tracking

- Each tracker now carries a run_id (UUIDv4) included in all emitted
  events, scoping every metric to a single execution
- At-most-once semantics: duplicate calls to track_duration,
  track_tokens, track_success/track_error, track_feedback, and
  track_time_to_first_token on the same tracker are dropped with a
  warning

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: jsonbailey

lib/server/ai/ai_config_tracker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'ldclient-rb'
+require 'securerandom'
 
 module LaunchDarkly
   module Server
@@ -64,6 +65,13 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p
           @provider_name = provider_name
           @context = context
           @summary = MetricSummary.new
+          @run_id = SecureRandom.uuid
+          @tracked_duration = false
+          @tracked_time_to_first_token = false
+          @tracked_tokens = false
+          @tracked_success = nil
+          @tracked_feedback = false
+          @logger = LaunchDarkly::Server::AI.default_logger
         end
 
         #
@@ -72,6 +80,11 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p
         # @param duration [Integer] The duration in milliseconds
         #
         def track_duration(duration)
+          if @tracked_duration
+            @logger&.warn("Duration has already been tracked for this execution.")
+            return
+          end
+          @tracked_duration = true
           @summary.duration = duration
           @ld_client.track(
             '$ld:ai:duration:total',
@@ -101,6 +114,11 @@ def track_duration_of(&block)
         # @param duration [Integer] The duration in milliseconds
         #
         def track_time_to_first_token(time_to_first_token)
+          if @tracked_time_to_first_token
+            @logger&.warn("Time to first token has already been tracked for this execution.")
+            return
+          end
+          @tracked_time_to_first_token = true
           @summary.time_to_first_token = time_to_first_token
           @ld_client.track(
             '$ld:ai:tokens:ttf',
@@ -116,6 +134,11 @@ def track_time_to_first_token(time_to_first_token)
         # @param kind [Symbol] The kind of feedback (:positive or :negative)
         #
         def track_feedback(kind:)
+          if @tracked_feedback
+            @logger&.warn("Feedback has already been tracked for this execution.")
+            return
+          end
+          @tracked_feedback = true
           @summary.feedback = kind
           event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative'
           @ld_client.track(
@@ -130,6 +153,11 @@ def track_feedback(kind:)
         # Track a successful AI generation
         #
         def track_success
+          unless @tracked_success.nil?
+            @logger&.warn("Success or error has already been tracked for this execution.")
+            return
+          end
+          @tracked_success = true
           @summary.success = true
           @ld_client.track(
             '$ld:ai:generation:success',
@@ -143,6 +171,11 @@ def track_success
         # Track an error in AI generation
         #
         def track_error
+          unless @tracked_success.nil?
+            @logger&.warn("Success or error has already been tracked for this execution.")
+            return
+          end
+          @tracked_success = false
           @summary.success = false
           @ld_client.track(
             '$ld:ai:generation:error',
@@ -158,6 +191,11 @@ def track_error
         # @param token_usage [TokenUsage] An object containing token usage details
         #
         def track_tokens(token_usage)
+          if @tracked_tokens
+            @logger&.warn("Tokens have already been tracked for this execution.")
+            return
+          end
+          @tracked_tokens = true
           @summary.usage = token_usage
           if token_usage.total.positive?
             @ld_client.track(
@@ -223,6 +261,7 @@ def track_bedrock_converse_metrics(&block)
 
         private def flag_data
           {
+            runId: @run_id,
             variationKey: @variation_key,
             configKey: @config_key,
             version: @version,

spec/server/ai/config_tracker_spec.rb

@@ -27,7 +27,7 @@
   end
 
   let(:context) { LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) }
-  let(:tracker_flag_data) { { variationKey: 'test-variation', configKey: 'test-config', version: 1, modelName: 'fakeModel', providerName: 'fakeProvider' } }
+  let(:tracker_flag_data) { { runId: kind_of(String), variationKey: 'test-variation', configKey: 'test-config', version: 1, modelName: 'fakeModel', providerName: 'fakeProvider' } }
   let(:tracker) do
     described_class.new(
       ld_client: ld_client,
@@ -322,25 +322,107 @@
       expect(tracker.summary.success).to be false
     end
 
-    it 'overwrites success with error if both are tracked' do
+    it 'does not track error if success has already been tracked' do
       expect(ld_client).to receive(:track).with(
         '$ld:ai:generation:success',
         context,
         tracker_flag_data,
         1
       )
+
+      tracker.track_success
+      expect(tracker.summary.success).to be true
+      tracker.track_error
+      expect(tracker.summary.success).to be true
+    end
+  end
+
+  describe 'at-most-once tracking' do
+    it 'only tracks duration once' do
       expect(ld_client).to receive(:track).with(
-        '$ld:ai:generation:error',
+        '$ld:ai:duration:total',
         context,
         tracker_flag_data,
-        1
-      )
+        100
+      ).once
+      tracker.track_duration(100)
+      tracker.track_duration(200)
+      expect(tracker.summary.duration).to eq(100)
+    end
+
+    it 'only tracks time to first token once' do
+      expect(ld_client).to receive(:track).with(
+        '$ld:ai:tokens:ttf',
+        context,
+        tracker_flag_data,
+        100
+      ).once
+      tracker.track_time_to_first_token(100)
+      tracker.track_time_to_first_token(200)
+      expect(tracker.summary.time_to_first_token).to eq(100)
+    end
+
+    it 'only tracks tokens once' do
+      tokens1 = LaunchDarkly::Server::AI::TokenUsage.new(total: 300, input: 200, output: 100)
+      tokens2 = LaunchDarkly::Server::AI::TokenUsage.new(total: 600, input: 400, output: 200)
+      expect(ld_client).to receive(:track).with(
+        '$ld:ai:tokens:total',
+        context,
+        tracker_flag_data,
+        300
+      ).once
+      expect(ld_client).to receive(:track).with(
+        '$ld:ai:tokens:input',
+        context,
+        tracker_flag_data,
+        200
+      ).once
+      expect(ld_client).to receive(:track).with(
+        '$ld:ai:tokens:output',
+        context,
+        tracker_flag_data,
+        100
+      ).once
+      tracker.track_tokens(tokens1)
+      tracker.track_tokens(tokens2)
+      expect(tracker.summary.usage).to eq(tokens1)
+    end
 
+    it 'only tracks success once' do
+      expect(ld_client).to receive(:track).with(
+        '$ld:ai:generation:success',
+        context,
+        tracker_flag_data,
+        1
+      ).once
+      tracker.track_success
       tracker.track_success
       expect(tracker.summary.success).to be true
+    end
+
+    it 'only tracks error once' do
+      expect(ld_client).to receive(:track).with(
+        '$ld:ai:generation:error',
+        context,
+        tracker_flag_data,
+        1
+      ).once
+      tracker.track_error
       tracker.track_error
       expect(tracker.summary.success).to be false
     end
+
+    it 'only tracks feedback once' do
+      expect(ld_client).to receive(:track).with(
+        '$ld:ai:feedback:user:positive',
+        context,
+        tracker_flag_data,
+        1
+      ).once
+      tracker.track_feedback(kind: :positive)
+      tracker.track_feedback(kind: :negative)
+      expect(tracker.summary.feedback).to eq(:positive)
+    end
   end
 
   describe '#summary' do
@@ -370,11 +452,14 @@
   end
 
   describe '#flag_data' do
-    it 'includes model_name and provider_name in flag data' do
-      expect(tracker.send(:flag_data)).to include(
+    it 'includes runId, model_name, and provider_name in flag data' do
+      flag_data = tracker.send(:flag_data)
+      expect(flag_data).to include(
+        runId: kind_of(String),
         modelName: 'fakeModel',
         providerName: 'fakeProvider'
       )
+      expect(flag_data[:runId]).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i)
     end
   end