chore: Add Agent and Judge field parsers to ConfigFactory (#279)

mattrmc1 · web-flow · commit 12c0b7a18624 · 2026-06-05T15:03:40.000-05:00
## Summary Adds the **parsing primitives** for agent and judge config fields to `ConfigFactory`, plus a small visibility fix on `LdAiConfigBase`. The only new public type is `ToolConfig`. Four new internal static helpers on `ConfigFactory`: - `ParseInstructions(LdValue)` — returns the string at `instructions`, or null if missing/wrong-typed (used by agent configs). - `ParseTools(LdValue)` — reads the `tools` object as a `IReadOnlyDictionary<string, ToolConfig>`, keyed by the wire-format property name. Returns an empty dictionary for missing/wrong-typed inputs (tolerant, consistent with the existing parsers). - `ParseJudgeConfiguration(LdValue)` — reads the `judgeConfiguration.judges[]` array into a `JudgeConfigurationData` of `(key, samplingRate)` entries. Returns null when the block is missing, an empty list when `judges` is missing/wrong-typed. - `ParseEvaluationMetricKey(LdValue)` — returns the string at `evaluationMetricKey`, or null. `ToolConfig` is the only new public type. It is a sealed record with an internal constructor and lives in `LaunchDarkly.Sdk.Server.Ai.Config` alongside `Message`, `ModelConfig`, and `ProviderConfig` — same convention as the rest of the data shapes. `JudgeConfigurationData` and `JudgeEntry` are internal-only; they're shaped to fit how the judge sampling logic will consume them. ## Visibility fix on `LdAiConfigBase` `VariationKey` and `Version` on `LdAiConfigBase` are now `internal` rather than `public`. The doc comment on both already said *"This field meant for internal LaunchDarkly usage"* — they were inadvertently exposed. This is a breaking change for any caller that was reading them off a concrete config (`LdAiCompletionConfig`, etc.), but no public API was ever expected to depend on them. ## Test plan - [ ] `dotnet build` succeeds across `netstandard2.0`, `net462`, `net8.0` - [ ] `dotnet test --framework net8.0` passes - [ ] New `ConfigFactoryParserTest` covers each parser: - `ParseTools_ReturnsTwoEntries_WithCorrectCustomParameters` - `ParseJudgeConfiguration_ReturnsTwoEntries_WithCorrectKeysAndSamplingRates` - `ParseEvaluationMetricKey_ReturnsParsedString` - `ParseInstructions_ReturnsRawUninterpolatedString` - `ParseAllFields_WithNoNewFields_ReturnsNullOrEmptyWithoutError` (tolerant fallback for missing blocks) - [ ] Reviewer confirms `ToolConfig` shape matches the cross-SDK pattern  --- > [!NOTE] > **Medium Risk** > Public API break on LdAiConfig.VariationKey/Version for external callers; new parsers are isolated but judge/tool shapes will underpin later agent/judge behavior. > > **Overview** > Adds **internal `ConfigFactory` parsers** for upcoming agent/judge config modes: `instructions`, root-level `tools`, `judgeConfiguration.judges`, and `evaluationMetricKey`, with tolerant skipping of malformed entries and **no wiring into build paths yet** (step 1 of 6). > > Introduces public **`LdAiConfigTypes.Tool`** and **`JudgeConfiguration`** (+ nested `Judge`) to hold parsed wire data; adds **`ConfigFactoryParserTest`** for happy paths, malformed inputs, and missing fields. > > **Breaking visibility change:** `LdAiConfig.VariationKey` and `Version` are now **`internal`** (still used inside the SDK for trackers), aligning with their “internal LaunchDarkly usage” docs. > > <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit e7858b0. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
diff --git a/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs b/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs
@@ -195,6 +195,52 @@ private static IReadOnlyDictionary<string, LdValue> LdValueObjectToDictionary(Ld
         return result;
     }
 
+    internal static string ParseInstructions(LdValue instructionsValue)
+    {
+        return instructionsValue.Type == LdValueType.String ? instructionsValue.AsString : null;
+    }
+
+    internal static IReadOnlyDictionary<string, LdAiConfigTypes.Tool> ParseTools(LdValue toolsValue)
+    {
+        if (toolsValue.Type != LdValueType.Object) return ImmutableDictionary<string, LdAiConfigTypes.Tool>.Empty;
+        var result = ImmutableDictionary.CreateBuilder<string, LdAiConfigTypes.Tool>();
+        foreach (var kv in toolsValue.Dictionary)
+        {
+            var tool = kv.Value;
+            if (tool.Type != LdValueType.Object) continue;
+            result[kv.Key] = new LdAiConfigTypes.Tool(
+                tool.Get("name").AsString ?? "",
+                tool.Get("description").AsString,
+                tool.Get("type").AsString,
+                LdValueObjectToDictionary(tool.Get("parameters")),
+                LdValueObjectToDictionary(tool.Get("customParameters")));
+        }
+        return result.ToImmutable();
+    }
+
+    internal static LdAiConfigTypes.JudgeConfiguration ParseJudgeConfiguration(LdValue judgeConfigurationValue)
+    {
+        if (judgeConfigurationValue.Type != LdValueType.Object) return null;
+        var judgesArray = judgeConfigurationValue.Get("judges");
+        if (judgesArray.Type != LdValueType.Array) return new LdAiConfigTypes.JudgeConfiguration(new List<LdAiConfigTypes.JudgeConfiguration.Judge>());
+        var entries = new List<LdAiConfigTypes.JudgeConfiguration.Judge>();
+        for (var i = 0; i < judgesArray.Count; i++)
+        {
+            var j = judgesArray.Get(i);
+            if (j.Type != LdValueType.Object) continue;
+            entries.Add(new LdAiConfigTypes.JudgeConfiguration.Judge(
+                j.Get("key").AsString ?? "",
+                j.Get("samplingRate").AsDouble));
+        }
+        return new LdAiConfigTypes.JudgeConfiguration(entries);
+    }
+
+    internal static string ParseEvaluationMetricKey(LdValue value)
+    {
+        var emk = value.Get("evaluationMetricKey");
+        return emk.Type == LdValueType.String ? emk.AsString : null;
+    }
+
     private static LdAiConfigTypes.Role ParseRole(string roleString)
     {
         // The wire format uses capitalized "User" / "System" / "Assistant"; Enum.TryParse with
diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs
@@ -33,12 +33,12 @@ public abstract class LdAiConfig
     /// <summary>
     /// This field meant for internal LaunchDarkly usage.
     /// </summary>
-    public string VariationKey { get; }
+    internal string VariationKey { get; }
 
     /// <summary>
     /// This field meant for internal LaunchDarkly usage.
     /// </summary>
-    public int Version { get; }
+    internal int Version { get; }
 
     /// <summary>
     /// Factory that produces a tracker for the config. The factory is mode-agnostic — it
diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs
@@ -98,4 +98,87 @@ internal ProviderConfig(string name)
             Name = name;
         }
     }
+
+    /// <summary>
+    /// Represents a tool available to the model in agent mode.
+    /// </summary>
+    public sealed record Tool
+    {
+        /// <summary>
+        /// The name of the tool.
+        /// </summary>
+        public readonly string Name;
+
+        /// <summary>
+        /// A description of the tool.
+        /// </summary>
+        public readonly string Description;
+
+        /// <summary>
+        /// The type of the tool.
+        /// </summary>
+        public readonly string Type;
+
+        /// <summary>
+        /// The tool's built-in parameters provided by LaunchDarkly.
+        /// </summary>
+        public readonly IReadOnlyDictionary<string, LdValue> Parameters;
+
+        /// <summary>
+        /// The tool's custom parameters provided by the user.
+        /// </summary>
+        public readonly IReadOnlyDictionary<string, LdValue> CustomParameters;
+
+        internal Tool(
+            string name,
+            string description,
+            string type,
+            IReadOnlyDictionary<string, LdValue> parameters,
+            IReadOnlyDictionary<string, LdValue> customParameters)
+        {
+            Name = name;
+            Description = description;
+            Type = type;
+            Parameters = parameters?.ToImmutableDictionary() ?? ImmutableDictionary<string, LdValue>.Empty;
+            CustomParameters = customParameters?.ToImmutableDictionary() ?? ImmutableDictionary<string, LdValue>.Empty;
+        }
+    }
+
+    /// <summary>
+    /// Configuration for the judges associated with an AI Config.
+    /// </summary>
+    public sealed class JudgeConfiguration
+    {
+        /// <summary>
+        /// Represents a single judge entry within a <see cref="JudgeConfiguration"/>.
+        /// </summary>
+        public sealed class Judge
+        {
+            /// <summary>
+            /// The key identifying this judge.
+            /// </summary>
+            public string Key { get; }
+
+            /// <summary>
+            /// The fraction of requests that this judge evaluates, in the range [0, 1].
+            /// </summary>
+            public double SamplingRate { get; }
+
+            internal Judge(string key, double samplingRate)
+            {
+                Key = key;
+                SamplingRate = samplingRate;
+            }
+        }
+
+        /// <summary>
+        /// The list of judges.
+        /// </summary>
+        public IReadOnlyList<Judge> Judges { get; }
+
+        internal JudgeConfiguration(IReadOnlyList<Judge> judges)
+        {
+            Judges = judges;
+        }
+    }
 }
diff --git a/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs b/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs
@@ -0,0 +1,207 @@
+using System.Collections.Generic;
+using LaunchDarkly.Sdk.Server.Ai.Config;
+using Xunit;
+
+namespace LaunchDarkly.Sdk.Server.Ai;
+
+public class ConfigFactoryParserTest
+{
+    [Fact]
+    public void ParseTools_ReturnsTwoEntries_WithCorrectCustomParameters()
+    {
+        var toolsValue = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+        {
+            ["search"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+            {
+                ["name"] = LdValue.Of("search"),
+                ["description"] = LdValue.Of("Searches the web"),
+                ["type"] = LdValue.Of("function"),
+                ["parameters"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+                {
+                    ["query"] = LdValue.Of("string")
+                }),
+                ["customParameters"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+                {
+                    ["timeout"] = LdValue.Of(30)
+                })
+            }),
+            ["calculator"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+            {
+                ["name"] = LdValue.Of("calculator"),
+                ["description"] = LdValue.Of("Performs arithmetic"),
+                ["type"] = LdValue.Of("function"),
+                ["parameters"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>()),
+                ["customParameters"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+                {
+                    ["precision"] = LdValue.Of(10)
+                })
+            })
+        });
+
+        // Build a root-level payload that also contains model.parameters.tools to confirm
+        // they remain separate and are not mixed with the root tools map.
+        var rootValue = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+        {
+            ["tools"] = toolsValue,
+            ["model"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+            {
+                ["parameters"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+                {
+                    ["tools"] = LdValue.ArrayOf(LdValue.Of("some-tool"))
+                })
+            })
+        });
+
+        var tools = ConfigFactory.ParseTools(rootValue.Get("tools"));
+
+        Assert.Equal(2, tools.Count);
+        Assert.True(tools.ContainsKey("search"));
+        Assert.True(tools.ContainsKey("calculator"));
+
+        Assert.Equal(30, tools["search"].CustomParameters["timeout"].AsInt);
+        Assert.Equal(10, tools["calculator"].CustomParameters["precision"].AsInt);
+        Assert.Equal("string", tools["search"].Parameters["query"].AsString);
+
+        // model.parameters.tools (opaque array) must be unaffected by root tools parsing.
+        var modelTools = rootValue.Get("model").Get("parameters").Get("tools");
+        Assert.Equal(LdValueType.Array, modelTools.Type);
+        Assert.Equal(1, modelTools.Count);
+    }
+
+    [Fact]
+    public void ParseJudgeConfiguration_ReturnsTwoEntries_WithCorrectKeysAndSamplingRates()
+    {
+        var rootValue = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+        {
+            ["judgeConfiguration"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+            {
+                ["judges"] = LdValue.ArrayOf(
+                    LdValue.ObjectFrom(new Dictionary<string, LdValue>
+                    {
+                        ["key"] = LdValue.Of("judge-relevance"),
+                        ["samplingRate"] = LdValue.Of(0.5)
+                    }),
+                    LdValue.ObjectFrom(new Dictionary<string, LdValue>
+                    {
+                        ["key"] = LdValue.Of("judge-coherence"),
+                        ["samplingRate"] = LdValue.Of(1.0)
+                    })
+                )
+            })
+        });
+
+        var judgeConfig = ConfigFactory.ParseJudgeConfiguration(rootValue.Get("judgeConfiguration"));
+
+        Assert.NotNull(judgeConfig);
+        Assert.Equal(2, judgeConfig.Judges.Count);
+        Assert.Equal("judge-relevance", judgeConfig.Judges[0].Key);
+        Assert.Equal(0.5, judgeConfig.Judges[0].SamplingRate);
+        Assert.Equal("judge-coherence", judgeConfig.Judges[1].Key);
+        Assert.Equal(1.0, judgeConfig.Judges[1].SamplingRate);
+    }
+
+    [Fact]
+    public void ParseEvaluationMetricKey_ReturnsParsedString()
+    {
+        var rootValue = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+        {
+            ["evaluationMetricKey"] = LdValue.Of("$ld:ai:judge:relevance")
+        });
+
+        var key = ConfigFactory.ParseEvaluationMetricKey(rootValue);
+
+        Assert.Equal("$ld:ai:judge:relevance", key);
+    }
+
+    [Fact]
+    public void ParseInstructions_ReturnsRawUninterpolatedString()
+    {
+        var rootValue = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+        {
+            ["instructions"] = LdValue.Of("You are a helpful assistant specializing in {{topic}}")
+        });
+
+        var instructions = ConfigFactory.ParseInstructions(rootValue.Get("instructions"));
+
+        Assert.Equal("You are a helpful assistant specializing in {{topic}}", instructions);
+    }
+
+    [Fact]
+    public void ParseTools_SkipsNonObjectValues()
+    {
+        var toolsValue = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+        {
+            ["good"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+            {
+                ["name"] = LdValue.Of("good"),
+                ["description"] = LdValue.Of("A valid tool"),
+                ["type"] = LdValue.Of("function"),
+                ["parameters"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>()),
+                ["customParameters"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>())
+            }),
+            ["bad-string"] = LdValue.Of("not-an-object"),
+            ["bad-number"] = LdValue.Of(42),
+            ["bad-array"] = LdValue.ArrayOf(LdValue.Of("x"))
+        });
+
+        var tools = ConfigFactory.ParseTools(toolsValue);
+
+        Assert.Single(tools);
+        Assert.True(tools.ContainsKey("good"));
+    }
+
+    [Fact]
+    public void ParseJudgeConfiguration_SkipsNonObjectElements()
+    {
+        var rootValue = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+        {
+            ["judgeConfiguration"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+            {
+                ["judges"] = LdValue.ArrayOf(
+                    LdValue.ObjectFrom(new Dictionary<string, LdValue>
+                    {
+                        ["key"] = LdValue.Of("judge-relevance"),
+                        ["samplingRate"] = LdValue.Of(0.5)
+                    }),
+                    LdValue.Of("not-an-object"),
+                    LdValue.Of(99),
+                    LdValue.ArrayOf(LdValue.Of("nested"))
+                )
+            })
+        });
+
+        var judgeConfig = ConfigFactory.ParseJudgeConfiguration(rootValue.Get("judgeConfiguration"));
+
+        Assert.NotNull(judgeConfig);
+        Assert.Single(judgeConfig.Judges);
+        Assert.Equal("judge-relevance", judgeConfig.Judges[0].Key);
+        Assert.Equal(0.5, judgeConfig.Judges[0].SamplingRate);
+    }
+
+    [Fact]
+    public void ParseAllFields_WithNoNewFields_ReturnsNullOrEmptyWithoutError()
+    {
+        var rootValue = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+        {
+            ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+            {
+                ["enabled"] = LdValue.Of(true),
+                ["variationKey"] = LdValue.Of("v1")
+            }),
+            ["model"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
+            {
+                ["name"] = LdValue.Of("some-model")
+            })
+        });
+
+        var instructions = ConfigFactory.ParseInstructions(rootValue.Get("instructions"));
+        var tools = ConfigFactory.ParseTools(rootValue.Get("tools"));
+        var judgeConfig = ConfigFactory.ParseJudgeConfiguration(rootValue.Get("judgeConfiguration"));
+        var evaluationMetricKey = ConfigFactory.ParseEvaluationMetricKey(rootValue);
+
+        Assert.Null(instructions);
+        Assert.Empty(tools);
+        Assert.Null(judgeConfig);
+        Assert.Null(evaluationMetricKey);
+    }
+}
