feat: add AI Config judge support (#345)

**Requirements**

- [X] I have added test coverage for new or changed functionality
- [X] I have followed the repository's [pull request submission
guidelines](../blob/v5/CONTRIBUTING.md#submitting-pull-requests)
- [X] I have validated my changes against all supported platform
versions

**Related issues**

See
https://docs.google.com/document/d/1lzYwQqCcTzN_2zkxJZDfJtgUcEJ4jbpx0KSsJ2bRENw/edit?tab=t.0#heading=h.5d8l30brvyuw
for context

For other SDK implementations, see:
- launchdarkly/js-core#1073
- launchdarkly/python-server-sdk-ai#86 &
launchdarkly/python-server-sdk-ai#64

**Describe the solution you've provided**

Extending the Go SDK to support AI Config evaluations. This includes
custom evaluator support as well.

This SDK was written with hopes to be congruent with the python and node
implementations. Changes were verified by a local app that was created;
[the resultant data can be observed in the evaluator metrics for this AI
config](https://ld-stg.launchdarkly.com/projects/default/ai-configs/kf-comp-feb-3/monitoring?from_ts=1770094800000&to_ts=1770353999999&env=staging&selected-env=staging&chartTypes=Tokens%2CSatisfaction%2CGenerations%2CTime+to+generate%2CError+rate%2CTime+to+first+token%2CCosts%2CEvaluator+metrics+%28avg%29).

**Describe alternatives you've considered**

Provide a clear and concise description of any alternative solutions or
features you've considered.

**Additional context**

Add any other context about the pull request here.

<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> **Medium Risk**
> Adds new evaluation and metric-tracking paths (including dynamic
metric keys and new event payload fields), which could affect analytics
correctness and runtime behavior if misconfigured. Changes are
well-covered by tests but touch core SDK tracking surfaces.
> 
> **Overview**
> Adds **judge-mode support** to AI Configs by extending the config
datamodel and builder with `mode`,
`evaluationMetricKey`/`evaluationMetricKeys`, and `judgeConfiguration`
(with defensive copying to keep configs immutable).
> 
> Introduces `Client.JudgeConfig` to fetch judge configs while
preserving `{{message_history}}` / `{{response_to_evaluate}}`
placeholders for a second Mustache interpolation pass during evaluation,
and adds a new `ldai/judge` package that samples, interpolates, invokes
a structured provider, and parses judge responses.
> 
> Extends `Tracker` with `TrackJudgeResponse` to emit evaluation scores
as metrics (including optional `judgeConfigKey` in event data), and adds
comprehensive tests covering parsing, placeholder preservation, schema
generation, sampling, and response validation.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
41141b9. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

Author: knfreemLD

ldai/client.go

@@ -71,7 +71,17 @@ func (c *Client) Config(
 	variables map[string]interface{},
 ) (Config, *Tracker) {
 	_ = c.sdk.TrackMetric("$ld:ai:config:function:single", context, 1, ldvalue.String(key))
+	return c.evaluateConfig(key, context, defaultValue, variables)
+}
 
+// evaluateConfig fetches and interpolates an AI Config without emitting any metric.
+// Callers (Config, JudgeConfig) are meant to emit their own metric before calling this.
+func (c *Client) evaluateConfig(
+	key string,
+	context ldcontext.Context,
+	defaultValue Config,
+	variables map[string]interface{},
+) (Config, *Tracker) {
 	result, _ := c.sdk.JSONVariation(key, context, defaultValue.AsLdValue())
 
 	// The spec requires the config to at least be an object (although all properties are optional, so it may be an
@@ -102,7 +112,11 @@ func (c *Client) Config(
 	builder := NewConfig().
 		WithModelName(parsed.Model.Name).
 		WithProviderName(parsed.Provider.Name).
-		WithEnabled(parsed.Meta.Enabled)
+		WithEnabled(parsed.Meta.Enabled).
+		WithMode(parsed.Mode).
+		WithEvaluationMetricKey(parsed.EvaluationMetricKey).
+		WithEvaluationMetricKeys(parsed.EvaluationMetricKeys).
+		WithJudgeConfiguration(parsed.JudgeConfiguration)
 
 	for k, v := range parsed.Model.Parameters {
 		builder.WithModelParam(k, v)
@@ -174,3 +188,38 @@ func interpolateTemplate(template string, variables map[string]interface{}) (str
 	}
 	return m.RenderString(variables)
 }
+
+// JudgeConfig evaluates an AI Config, tracking it as a judge function. See Config for details.
+//
+// This method extends the provided variables with reserved judge variables:
+// - "message_history": "{{message_history}}"
+// - "response_to_evaluate": "{{response_to_evaluate}}"
+//
+// These literal placeholder strings preserve the Mustache templates through the first interpolation
+// (during config fetch), allowing Judge.Evaluate() to perform a second interpolation with actual values.
+func (c *Client) JudgeConfig(
+	key string,
+	context ldcontext.Context,
+	defaultValue Config,
+	variables map[string]interface{},
+) (Config, *Tracker) {
+	_ = c.sdk.TrackMetric("$ld:ai:judge:function:single", context, 1, ldvalue.String(key))
+
+	// Extend variables with reserved judge placeholders
+	extendedVariables := make(map[string]interface{})
+	for k, v := range variables {
+		// Warn if user tries to override reserved variables
+		if k == "message_history" || k == "response_to_evaluate" {
+			c.logger.Warnf("AI Config '%s': variable '%s' is reserved by judge and will be ignored", key, k)
+			continue
+		}
+		extendedVariables[k] = v
+	}
+
+	// Inject reserved variables as literal placeholder strings
+	// These will be preserved through the first interpolation and resolved during Judge.Evaluate()
+	extendedVariables["message_history"] = "{{message_history}}"
+	extendedVariables["response_to_evaluate"] = "{{response_to_evaluate}}"
+
+	return c.evaluateConfig(key, context, defaultValue, extendedVariables)
+}

ldai/client_test.go

@@ -329,6 +329,42 @@ func TestConfigMethodTracking(t *testing.T) {
 	assert.ElementsMatch(t, expectedEvents, mockSDK.events)
 }
 
+// TestJudgeConfigMethodTracking verifies that JudgeConfig emits only the judge metric,
+// not the config metric, so judge evaluations are not double-counted on the dashboard.
+func TestJudgeConfigMethodTracking(t *testing.T) {
+	json := []byte(`{
+		"_ldMeta": {"variationKey": "1", "enabled": true},
+		"mode": "judge",
+		"evaluationMetricKey": "toxicity",
+		"messages": [{"content": "test", "role": "system"}]
+	}`)
+	mockSDK := newMockSDK(json, nil)
+	client, err := NewClient(mockSDK)
+	require.NoError(t, err)
+	require.NotNil(t, client)
+
+	defaultConfig := Disabled()
+	context := ldcontext.New("user-key")
+	configKey := "judge-config-key"
+
+	config, tracker := client.JudgeConfig(configKey, context, defaultConfig, nil)
+
+	require.NotNil(t, config)
+	require.NotNil(t, tracker)
+
+	// Only the judge metric should be emitted; evaluateConfig does not emit any metric.
+	expectedEvents := []mockEvent{
+		{
+			eventName:   "$ld:ai:judge:function:single",
+			context:     context,
+			metricValue: 1,
+			data:        ldvalue.String(configKey),
+		},
+	}
+	assert.ElementsMatch(t, expectedEvents, mockSDK.events,
+		"JudgeConfig must not emit $ld:ai:config:function:single to avoid double-counting")
+}
+
 func TestCanSetModelParameters(t *testing.T) {
 	client, err := NewClient(newMockSDK(nil, nil))
 	require.NoError(t, err)
@@ -579,3 +615,167 @@ func TestInterpolation(t *testing.T) {
 		assert.Equal(t, "user_kind=<>,cat_kind=<>", result)
 	})
 }
+
+func TestParseJudgeSpecificFields(t *testing.T) {
+	json := []byte(`{
+		"_ldMeta": {"variationKey": "1", "enabled": true},
+		"mode": "judge",
+		"evaluationMetricKey": "toxicity",
+		"judgeConfiguration": {
+			"judges": [
+				{"key": "judge1", "samplingRate": 0.5},
+				{"key": "judge2", "samplingRate": 1.0}
+			]
+		},
+		"messages": [
+			{"content": "test", "role": "system"}
+		]
+	}`)
+
+	client, err := NewClient(newMockSDK(json, nil))
+	require.NoError(t, err)
+	require.NotNil(t, client)
+
+	cfg, _ := client.Config("key", ldcontext.New("user"), Disabled(), nil)
+
+	assert.Equal(t, "judge", cfg.Mode())
+	assert.Equal(t, "toxicity", cfg.EvaluationMetricKey())
+
+	judgeConfig := cfg.JudgeConfiguration()
+	require.NotNil(t, judgeConfig)
+	require.Len(t, judgeConfig.Judges, 2)
+	assert.Equal(t, "judge1", judgeConfig.Judges[0].Key)
+	assert.Equal(t, 0.5, judgeConfig.Judges[0].SamplingRate)
+	assert.Equal(t, "judge2", judgeConfig.Judges[1].Key)
+	assert.Equal(t, 1.0, judgeConfig.Judges[1].SamplingRate)
+}
+
+func TestParseEvaluationMetricKeys(t *testing.T) {
+	json := []byte(`{
+		"_ldMeta": {"variationKey": "1", "enabled": true},
+		"mode": "judge",
+		"evaluationMetricKeys": ["relevance", "accuracy"],
+		"messages": [
+			{"content": "test", "role": "system"}
+		]
+	}`)
+
+	client, err := NewClient(newMockSDK(json, nil))
+	require.NoError(t, err)
+	require.NotNil(t, client)
+
+	cfg, _ := client.Config("key", ldcontext.New("user"), Disabled(), nil)
+
+	assert.Equal(t, "judge", cfg.Mode())
+	assert.Equal(t, "", cfg.EvaluationMetricKey())
+	assert.Equal(t, []string{"relevance", "accuracy"}, cfg.EvaluationMetricKeys())
+}
+
+func TestParseEvaluationMetricKeyPriority(t *testing.T) {
+	json := []byte(`{
+		"_ldMeta": {"variationKey": "1", "enabled": true},
+		"mode": "judge",
+		"evaluationMetricKey": "toxicity",
+		"evaluationMetricKeys": ["relevance", "accuracy"],
+		"messages": [
+			{"content": "test", "role": "system"}
+		]
+	}`)
+
+	client, err := NewClient(newMockSDK(json, nil))
+	require.NoError(t, err)
+	require.NotNil(t, client)
+
+	cfg, _ := client.Config("key", ldcontext.New("user"), Disabled(), nil)
+
+	assert.Equal(t, "judge", cfg.Mode())
+	// Both fields should be parsed
+	assert.Equal(t, "toxicity", cfg.EvaluationMetricKey())
+	assert.Equal(t, []string{"relevance", "accuracy"}, cfg.EvaluationMetricKeys())
+}
+
+func TestJudgeConfigurationImmutable(t *testing.T) {
+	// Test that mutations to JudgeConfiguration don't affect the Config
+	judgeConfig := &datamodel.JudgeConfiguration{
+		Judges: []datamodel.Judge{
+			{Key: "judge1", SamplingRate: 0.5},
+			{Key: "judge2", SamplingRate: 1.0},
+		},
+	}
+
+	builder := NewConfig().
+		Enable().
+		WithJudgeConfiguration(judgeConfig)
+	cfg := builder.Build()
+
+	// Mutate the original
+	judgeConfig.Judges[0].Key = "mutated"
+	judgeConfig.Judges = append(judgeConfig.Judges, datamodel.Judge{Key: "judge3", SamplingRate: 0.3})
+
+	// Config should not be affected
+	retrieved := cfg.JudgeConfiguration()
+	require.NotNil(t, retrieved)
+	require.Len(t, retrieved.Judges, 2)
+	assert.Equal(t, "judge1", retrieved.Judges[0].Key) // Should still be original value
+	assert.Equal(t, "judge2", retrieved.Judges[1].Key)
+
+	// Mutate the retrieved config
+	retrieved.Judges[0].Key = "mutated_again"
+	retrieved.Judges = append(retrieved.Judges, datamodel.Judge{Key: "judge4", SamplingRate: 0.4})
+
+	// Config should still not be affected
+	retrieved2 := cfg.JudgeConfiguration()
+	require.NotNil(t, retrieved2)
+	require.Len(t, retrieved2.Judges, 2)
+	assert.Equal(t, "judge1", retrieved2.Judges[0].Key) // Should still be original value
+	assert.Equal(t, "judge2", retrieved2.Judges[1].Key)
+}
+
+// TestJudgeConfig_PreservesReservedPlaceholders verifies that JudgeConfig injects reserved variables
+// so that {{message_history}} and {{response_to_evaluate}} are preserved for the second interpolation
+// pass during Judge.Evaluate(). Without this, Config's first Mustache pass would render them as empty.
+func TestJudgeConfig_PreservesReservedPlaceholders(t *testing.T) {
+	json := []byte(`{
+		"_ldMeta": {"variationKey": "1", "enabled": true},
+		"mode": "judge",
+		"evaluationMetricKey": "toxicity",
+		"messages": [
+			{"content": "You are a judge.", "role": "system"},
+			{"content": "Input: {{message_history}}\nOutput: {{response_to_evaluate}}", "role": "user"}
+		]
+	}`)
+
+	client, err := NewClient(newMockSDK(json, nil))
+	require.NoError(t, err)
+	require.NotNil(t, client)
+
+	cfg, _ := client.JudgeConfig("judge-key", ldcontext.New("user"), Disabled(), nil)
+
+	msgs := cfg.Messages()
+	require.Len(t, msgs, 2)
+	assert.Equal(t, "You are a judge.", msgs[0].Content)
+	assert.Contains(t, msgs[1].Content, "{{message_history}}", "JudgeConfig must preserve placeholder for second interpolation")
+	assert.Contains(t, msgs[1].Content, "{{response_to_evaluate}}", "JudgeConfig must preserve placeholder for second interpolation")
+	assert.Equal(t, "Input: {{message_history}}\nOutput: {{response_to_evaluate}}", msgs[1].Content)
+}
+
+// TestConfig_WithoutReservedVarsWipesJudgePlaceholders documents that Config (without reserved vars)
+// renders {{message_history}} and {{response_to_evaluate}} as empty when used for judge templates.
+func TestConfig_WithoutReservedVarsWipesJudgePlaceholders(t *testing.T) {
+	json := []byte(`{
+		"_ldMeta": {"variationKey": "1", "enabled": true},
+		"messages": [
+			{"content": "Input: {{message_history}}\nOutput: {{response_to_evaluate}}", "role": "user"}
+		]
+	}`)
+
+	client, err := NewClient(newMockSDK(json, nil))
+	require.NoError(t, err)
+	require.NotNil(t, client)
+
+	cfg, _ := client.Config("key", ldcontext.New("user"), Disabled(), nil)
+
+	msgs := cfg.Messages()
+	require.Len(t, msgs, 1)
+	assert.Equal(t, "Input: \nOutput: ", msgs[0].Content, "Config without reserved vars renders placeholders as empty")
+}

ldai/config.go

@@ -60,6 +60,33 @@ func (c *Config) CustomModelParam(key string) (ldvalue.Value, bool) {
 	return val, ok
 }
 
+// Mode returns the AI Config mode (e.g., "completion", "agent", "judge").
+func (c *Config) Mode() string {
+	return c.c.Mode
+}
+
+// EvaluationMetricKey returns the evaluation metric key for judge mode configs.
+func (c *Config) EvaluationMetricKey() string {
+	return c.c.EvaluationMetricKey
+}
+
+// EvaluationMetricKeys returns the deprecated array of evaluation metric keys.
+// Use EvaluationMetricKey instead.
+func (c *Config) EvaluationMetricKeys() []string {
+	return slices.Clone(c.c.EvaluationMetricKeys)
+}
+
+// JudgeConfiguration returns the judge configuration attached to this config, if any.
+// Returns a defensive copy to prevent mutations.
+func (c *Config) JudgeConfiguration() *datamodel.JudgeConfiguration {
+	if c.c.JudgeConfiguration == nil {
+		return nil
+	}
+	return &datamodel.JudgeConfiguration{
+		Judges: slices.Clone(c.c.JudgeConfiguration.Judges),
+	}
+}
+
 // AsLdValue is used internally.
 func (c *Config) AsLdValue() ldvalue.Value {
 	return ldvalue.FromJSONMarshal(c.c)
@@ -68,12 +95,16 @@ func (c *Config) AsLdValue() ldvalue.Value {
 // ConfigBuilder is used to define a default AI Config, returned when LaunchDarkly is unreachable or there
 // is an error evaluating the Config.
 type ConfigBuilder struct {
-	messages          []datamodel.Message
-	enabled           bool
-	providerName      string
-	modelName         string
-	modelParams       map[string]ldvalue.Value
-	modelCustomParams map[string]ldvalue.Value
+	messages             []datamodel.Message
+	enabled              bool
+	providerName         string
+	modelName            string
+	modelParams          map[string]ldvalue.Value
+	modelCustomParams    map[string]ldvalue.Value
+	mode                 string
+	evaluationMetricKey  string
+	evaluationMetricKeys []string
+	judgeConfiguration   *datamodel.JudgeConfiguration
 }
 
 // NewConfig returns a new ConfigBuilder. By default, the Config is disabled.
@@ -141,8 +172,47 @@ func (cb *ConfigBuilder) WithCustomModelParam(key string, value ldvalue.Value) *
 	return cb
 }
 
+// WithMode sets the AI Config mode (e.g., "completion", "agent", "judge").
+func (cb *ConfigBuilder) WithMode(mode string) *ConfigBuilder {
+	cb.mode = mode
+	return cb
+}
+
+// WithEvaluationMetricKey sets the evaluation metric key for judge mode configs.
+func (cb *ConfigBuilder) WithEvaluationMetricKey(key string) *ConfigBuilder {
+	cb.evaluationMetricKey = key
+	return cb
+}
+
+// WithEvaluationMetricKeys sets the deprecated array of evaluation metric keys.
+// Use WithEvaluationMetricKey instead.
+func (cb *ConfigBuilder) WithEvaluationMetricKeys(keys []string) *ConfigBuilder {
+	cb.evaluationMetricKeys = slices.Clone(keys)
+	return cb
+}
+
+// WithJudgeConfiguration sets the judge configuration for this config.
+// The provided judgeConfig is defensively copied.
+func (cb *ConfigBuilder) WithJudgeConfiguration(judgeConfig *datamodel.JudgeConfiguration) *ConfigBuilder {
+	if judgeConfig == nil {
+		cb.judgeConfiguration = nil
+		return cb
+	}
+	cb.judgeConfiguration = &datamodel.JudgeConfiguration{
+		Judges: slices.Clone(judgeConfig.Judges),
+	}
+	return cb
+}
+
 // Build creates a Config from the current builder state.
 func (cb *ConfigBuilder) Build() Config {
+	var judgeConfig *datamodel.JudgeConfiguration
+	if cb.judgeConfiguration != nil {
+		judgeConfig = &datamodel.JudgeConfiguration{
+			Judges: slices.Clone(cb.judgeConfiguration.Judges),
+		}
+	}
+
 	return Config{
 		c: datamodel.Config{
 			Messages: slices.Clone(cb.messages),
@@ -157,6 +227,10 @@ func (cb *ConfigBuilder) Build() Config {
 			Provider: datamodel.Provider{
 				Name: cb.providerName,
 			},
+			Mode:                 cb.mode,
+			EvaluationMetricKey:  cb.evaluationMetricKey,
+			EvaluationMetricKeys: slices.Clone(cb.evaluationMetricKeys),
+			JudgeConfiguration:   judgeConfig,
 		},
 	}
 }