feat!: Add per-execution runId, at-most-once tracking, and cross-process tracker resumption

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: jsonbailey

launchdarkly-server-sdk-ai.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
   spec.required_ruby_version = '>= 3.1.0'
 
+  spec.add_dependency 'base64'
   spec.add_dependency 'launchdarkly-server-sdk', '~> 8.5'
   spec.add_dependency 'logger'
   spec.add_dependency 'mustache', '~> 1.1'

lib/server/ai/ai_config_tracker.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'base64'
+require 'json'
 require 'ldclient-rb'
 require 'securerandom'
 
@@ -56,7 +58,7 @@ class AIConfigTracker
         # @param provider_name [String] The name of the AI provider
         # @param context [LDContext] The context used for the flag evaluation
         #
-        def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, provider_name:, context:)
+        def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, provider_name:, context:, run_id: nil)
           @ld_client = ld_client
           @variation_key = variation_key
           @config_key = config_key
@@ -65,7 +67,7 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p
           @provider_name = provider_name
           @context = context
           @summary = MetricSummary.new
-          @run_id = SecureRandom.uuid
+          @run_id = run_id || SecureRandom.uuid
           @tracked_duration = false
           @tracked_time_to_first_token = false
           @tracked_tokens = false
@@ -74,6 +76,49 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p
           @logger = LaunchDarkly::Server::AI.default_logger
         end
 
+        #
+        # Returns a URL-safe Base64-encoded JSON token that can be used to reconstruct
+        # a tracker in a different process (e.g. for deferred feedback).
+        #
+        # The token contains: runId, configKey, variationKey, version.
+        # modelName and providerName are NOT included.
+        #
+        # @return [String] the resumption token
+        #
+        def resumption_token
+          payload = {
+            runId: @run_id,
+            configKey: @config_key,
+            variationKey: @variation_key,
+            version: @version,
+          }
+          Base64.urlsafe_encode64(JSON.generate(payload), padding: false)
+        end
+
+        #
+        # Reconstructs a tracker from a resumption token.
+        #
+        # @param token [String] A URL-safe Base64-encoded JSON resumption token
+        # @param ld_client [LDClient] The LaunchDarkly client instance
+        # @param context [LDContext] The context for track events
+        # @return [AIConfigTracker] A new tracker instance
+        #
+        def self.from_resumption_token(token:, ld_client:, context:)
+          json = Base64.urlsafe_decode64(token)
+          payload = JSON.parse(json)
+
+          new(
+            ld_client: ld_client,
+            run_id: payload['runId'],
+            config_key: payload['configKey'],
+            variation_key: payload.fetch('variationKey', ''),
+            version: payload['version'],
+            model_name: '',
+            provider_name: '',
+            context: context
+          )
+        end
+
         #
         # Track the duration of an AI operation
         #

lib/server/ai/client.rb

@@ -103,16 +103,26 @@ def to_h
       # The AIConfig class represents an AI configuration.
       #
       class AIConfig
-        attr_reader :enabled, :messages, :tracker, :model, :provider
+        attr_reader :enabled, :messages, :model, :provider
 
-        def initialize(enabled: nil, model: nil, messages: nil, tracker: nil, provider: nil)
+        def initialize(enabled: nil, model: nil, messages: nil, tracker_factory: nil, provider: nil)
           @enabled = enabled
           @messages = messages
-          @tracker = tracker
+          @tracker_factory = tracker_factory
           @model = model
           @provider = provider
         end
 
+        #
+        # Creates a new tracker with a fresh runId for tracking a single AI execution.
+        # Returns nil when the config has no tracker factory (e.g. a static disabled config).
+        #
+        # @return [AIConfigTracker, nil] a new tracker instance, or nil
+        #
+        def create_tracker
+          @tracker_factory&.call
+        end
+
         #
         # Returns a new disabled AIConfig instance.
         #
@@ -182,6 +192,18 @@ def completion_config(key:, context:, default: nil, variables: nil)
           _completion_config(key:, context:, default: default || AIConfig.disabled, variables:)
         end
 
+        #
+        # Reconstructs a tracker from a resumption token, allowing deferred tracking
+        # (e.g. feedback from a different process).
+        #
+        # @param token [String] A resumption token obtained from AIConfigTracker#resumption_token
+        # @param context [LDContext] The context for track events
+        # @return [AIConfigTracker] A new tracker instance
+        #
+        def create_tracker(token:, context:)
+          AIConfigTracker.from_resumption_token(token: token, ld_client: @ld_client, context: context)
+        end
+
         # @deprecated Use {#completion_config} instead.
         def config(key:, context:, default: nil, variables: nil)
           warn '[DEPRECATION] `config` is deprecated. Use `completion_config` instead.'
@@ -226,20 +248,27 @@ def _completion_config(key:, context:, default:, variables: nil)
             )
           end
 
-          tracker = LaunchDarkly::Server::AI::AIConfigTracker.new(
-            ld_client: @ld_client,
-            variation_key: variation.dig(:_ldMeta, :variationKey) || '',
-            config_key: key,
-            version: variation.dig(:_ldMeta, :version) || 1,
-            model_name: model&.name || '',
-            provider_name: provider_config&.name || '',
-            context:
-          )
+          variation_key = variation.dig(:_ldMeta, :variationKey) || ''
+          version = variation.dig(:_ldMeta, :version) || 1
+          model_name = model&.name || ''
+          provider_name = provider_config&.name || ''
+
+          tracker_factory = lambda {
+            LaunchDarkly::Server::AI::AIConfigTracker.new(
+              ld_client: @ld_client,
+              variation_key: variation_key,
+              config_key: key,
+              version: version,
+              model_name: model_name,
+              provider_name: provider_name,
+              context: context
+            )
+          }
 
           AIConfig.new(
             enabled: variation.dig(:_ldMeta, :enabled) || false,
             messages:,
-            tracker:,
+            tracker_factory:,
             model:,
             provider: provider_config
           )

spec/server/ai/client_spec.rb

@@ -227,8 +227,9 @@
 
         expect(config.provider).not_to be_nil
         expect(config.provider.name).to eq('fakeProvider')
-        expect(config.tracker).not_to be_nil
-        expect(config.tracker.send(:flag_data)).to include(
+        tracker = config.create_tracker
+        expect(tracker).not_to be_nil
+        expect(tracker.send(:flag_data)).to include(
           modelName: 'fakeModel',
           providerName: 'fakeProvider'
         )
@@ -353,8 +354,9 @@
         expect(config.model).to be_nil
         expect(config.messages).to be_nil
         expect(config.provider).to be_nil
-        expect(config.tracker).not_to be_nil
-        expect(config.tracker.send(:flag_data)).to include(
+        tracker = config.create_tracker
+        expect(tracker).not_to be_nil
+        expect(tracker.send(:flag_data)).to include(
           modelName: '',
           providerName: ''
         )
@@ -375,6 +377,74 @@
 
         expect(config.enabled).to be false
       end
+
+      it 'create_tracker returns a new tracker with a fresh runId each time' do
+        context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' })
+        config = ai_client.completion_config(key: 'model-config', context:, variables: { 'name' => 'World' })
+
+        tracker1 = config.create_tracker
+        tracker2 = config.create_tracker
+
+        expect(tracker1).to be_a(LaunchDarkly::Server::AI::AIConfigTracker)
+        expect(tracker2).to be_a(LaunchDarkly::Server::AI::AIConfigTracker)
+        expect(tracker1).not_to equal(tracker2)
+
+        run_id1 = tracker1.send(:flag_data)[:runId]
+        run_id2 = tracker2.send(:flag_data)[:runId]
+        expect(run_id1).not_to eq(run_id2)
+        expect(run_id1).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i)
+        expect(run_id2).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i)
+      end
+
+      it 'create_tracker preserves flag metadata across calls' do
+        context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' })
+        config = ai_client.completion_config(key: 'model-config', context:, variables: { 'name' => 'World' })
+
+        tracker = config.create_tracker
+        flag_data = tracker.send(:flag_data)
+
+        expect(flag_data[:configKey]).to eq('model-config')
+        expect(flag_data[:variationKey]).to eq('abcd')
+        expect(flag_data[:version]).to eq(1)
+        expect(flag_data[:modelName]).to eq('fakeModel')
+        expect(flag_data[:providerName]).to eq('fakeProvider')
+      end
+
+      it 'create_tracker returns nil for static disabled config' do
+        config = LaunchDarkly::Server::AI::AIConfig.disabled
+        expect(config.create_tracker).to be_nil
+      end
+
+      it 'round-trips a tracker through a resumption token via client create_tracker' do
+        context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' })
+        config = ai_client.completion_config(key: 'model-config', context:, variables: { 'name' => 'World' })
+
+        tracker = config.create_tracker
+        token = tracker.resumption_token
+        original_run_id = tracker.send(:flag_data)[:runId]
+
+        restored = ai_client.create_tracker(token: token, context: context)
+
+        expect(restored).to be_a(LaunchDarkly::Server::AI::AIConfigTracker)
+        expect(restored.send(:flag_data)[:runId]).to eq(original_run_id)
+        expect(restored.send(:flag_data)[:configKey]).to eq('model-config')
+        expect(restored.send(:flag_data)[:modelName]).to eq('')
+        expect(restored.send(:flag_data)[:providerName]).to eq('')
+      end
+
+      it 'each tracker has independent at-most-once tracking' do
+        context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' })
+        config = ai_client.completion_config(key: 'model-config', context:, variables: { 'name' => 'World' })
+
+        tracker1 = config.create_tracker
+        tracker2 = config.create_tracker
+
+        tracker1.track_duration(100)
+        tracker2.track_duration(200)
+
+        expect(tracker1.summary.duration).to eq(100)
+        expect(tracker2.summary.duration).to eq(200)
+      end
     end
 
     describe LaunchDarkly::Server::AI::AIConfig do

spec/server/ai/config_tracker_spec.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'base64'
+require 'json'
 require 'launchdarkly-server-sdk'
 require 'launchdarkly-server-sdk-ai'
 
@@ -463,6 +465,94 @@
     end
   end
 
+  describe '#resumption_token' do
+    it 'returns a URL-safe base64-encoded JSON string' do
+      token = tracker.resumption_token
+      decoded = JSON.parse(Base64.urlsafe_decode64(token))
+
+      expect(decoded['runId']).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i)
+      expect(decoded['configKey']).to eq('test-config')
+      expect(decoded['variationKey']).to eq('test-variation')
+      expect(decoded['version']).to eq(1)
+    end
+
+    it 'does not include modelName or providerName' do
+      token = tracker.resumption_token
+      decoded = JSON.parse(Base64.urlsafe_decode64(token))
+
+      expect(decoded).not_to have_key('modelName')
+      expect(decoded).not_to have_key('providerName')
+    end
+
+    it 'contains the same runId as the tracker flag data' do
+      flag_data = tracker.send(:flag_data)
+      token = tracker.resumption_token
+      decoded = JSON.parse(Base64.urlsafe_decode64(token))
+
+      expect(decoded['runId']).to eq(flag_data[:runId])
+    end
+  end
+
+  describe '.from_resumption_token' do
+    it 'reconstructs a tracker that uses the same runId' do
+      original_token = tracker.resumption_token
+      original_run_id = tracker.send(:flag_data)[:runId]
+
+      restored = described_class.from_resumption_token(
+        token: original_token,
+        ld_client: ld_client,
+        context: context
+      )
+
+      expect(restored.send(:flag_data)[:runId]).to eq(original_run_id)
+      expect(restored.send(:flag_data)[:configKey]).to eq('test-config')
+      expect(restored.send(:flag_data)[:variationKey]).to eq('test-variation')
+      expect(restored.send(:flag_data)[:version]).to eq(1)
+    end
+
+    it 'sets modelName and providerName to empty strings' do
+      token = tracker.resumption_token
+
+      restored = described_class.from_resumption_token(
+        token: token,
+        ld_client: ld_client,
+        context: context
+      )
+
+      expect(restored.send(:flag_data)[:modelName]).to eq('')
+      expect(restored.send(:flag_data)[:providerName]).to eq('')
+    end
+
+    it 'can track events with the restored tracker' do
+      token = tracker.resumption_token
+      original_run_id = tracker.send(:flag_data)[:runId]
+
+      restored = described_class.from_resumption_token(
+        token: token,
+        ld_client: ld_client,
+        context: context
+      )
+
+      expected_data = {
+        runId: original_run_id,
+        variationKey: 'test-variation',
+        configKey: 'test-config',
+        version: 1,
+        modelName: '',
+        providerName: '',
+      }
+
+      expect(ld_client).to receive(:track).with(
+        '$ld:ai:feedback:user:positive',
+        context,
+        expected_data,
+        1
+      )
+
+      restored.track_feedback(kind: :positive)
+    end
+  end
+
   describe 'completion_config method tracking' do
     it 'calls track with correct parameters when completion_config is called' do
       allow(ld_client).to receive(:track)