chore: merge main into branch, resolve client.py conflicts

Both branches independently added evaluator/judge logic (this branch)
and root-level tools map support (main). Conflicts in _completion_config
and __evaluate_agent resolved by keeping both changes. Parameter order
swap for track_metrics_of_async auto-resolved.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: jsonbailey

packages/sdk/server-ai/src/ldai/__init__.py

@@ -27,6 +27,7 @@
     LDAIAgentConfig,
     LDAIAgentDefaults,
     LDMessage,
+    LDTool,
     ModelConfig,
     ProviderConfig,
 )
@@ -66,6 +67,7 @@
     'Judge',
     'JudgeConfiguration',
     'JudgeResult',
+    'LDTool',
     'LDMessage',
     'ModelConfig',
     'ProviderConfig',

packages/sdk/server-ai/src/ldai/client.py

@@ -25,6 +25,7 @@
     Edge,
     JudgeConfiguration,
     LDMessage,
+    LDTool,
     ModelConfig,
     ProviderConfig,
 )
@@ -51,6 +52,25 @@
 _DISABLED_JUDGE_DEFAULT = AIJudgeConfigDefault.disabled()
 
 
+def _parse_tools(tools_data: Optional[Dict[str, Any]]) -> Optional[Dict[str, LDTool]]:
+    """Parse the root-level tools map from a flag variation dict."""
+    if not isinstance(tools_data, dict):
+        return None
+    result = {}
+    for tool_name, tool_dict in tools_data.items():
+        if not isinstance(tool_dict, dict):
+            log.warning('Skipping tool "%s": expected a dict, got %s', tool_name, type(tool_dict).__name__)
+            continue
+        result[tool_name] = LDTool(
+            name=tool_dict.get('name', tool_name),
+            description=tool_dict.get('description'),
+            type=tool_dict.get('type'),
+            parameters=tool_dict.get('parameters'),
+            custom_parameters=tool_dict.get('customParameters'),
+        )
+    return result or None
+
+
 class LDAIClient:
     """The LaunchDarkly AI SDK client object."""
 
@@ -91,11 +111,12 @@ def _completion_config(
         default_ai_provider: Optional[str] = None,
     ) -> AICompletionConfig:
         (model, provider, messages, instructions,
-         tracker_factory, enabled, judge_configuration, _) = self.__evaluate(
+         tracker_factory, enabled, judge_configuration, variation) = self.__evaluate(
             key, context, default.to_dict(), variables
         )
 
         evaluator = self._build_evaluator(judge_configuration, context, default_ai_provider, variables)
+        tools = _parse_tools(variation.get('tools'))
 
         config = AICompletionConfig(
             key=key,
@@ -106,6 +127,7 @@ def _completion_config(
             create_tracker=tracker_factory,
             evaluator=evaluator,
             judge_configuration=judge_configuration,
+            tools=tools,
         )
 
         return config
@@ -912,7 +934,7 @@ def __evaluate_agent(
         :return: Configured AIAgentConfig instance.
         """
         (model, provider, messages, instructions,
-         tracker_factory, enabled, judge_configuration, _) = self.__evaluate(
+         tracker_factory, enabled, judge_configuration, variation) = self.__evaluate(
             key, context, default.to_dict(), variables, graph_key=graph_key
         )
 
@@ -922,6 +944,7 @@ def __evaluate_agent(
         effective_judge_configuration = judge_configuration or JudgeConfiguration(judges=[])
 
         evaluator = self._build_evaluator(effective_judge_configuration, context, default_ai_provider, variables)
+        tools = _parse_tools(variation.get('tools'))
 
         return AIAgentConfig(
             key=key,
@@ -932,6 +955,7 @@ def __evaluate_agent(
             create_tracker=tracker_factory,
             evaluator=evaluator,
             judge_configuration=effective_judge_configuration,
+            tools=tools,
         )
 
     def __interpolate_template(self, template: str, variables: Dict[str, Any]) -> str:

packages/sdk/server-ai/src/ldai/judge/__init__.py

@@ -75,8 +75,8 @@ async def evaluate(
             assert self._evaluation_response_structure is not None
 
             response = await tracker.track_metrics_of_async(
-                lambda: self._model_runner.invoke_structured_model(messages, self._evaluation_response_structure),
                 lambda result: result.metrics,
+                lambda: self._model_runner.invoke_structured_model(messages, self._evaluation_response_structure),
             )
 
             parsed = self._parse_evaluation_response(response.data)

packages/sdk/server-ai/src/ldai/managed_agent.py

@@ -29,8 +29,8 @@ async def run(self, input: str) -> AgentResult:
         """
         tracker = self._ai_config.create_tracker()
         return await tracker.track_metrics_of_async(
-            lambda: self._agent_runner.run(input),
             lambda result: result.metrics,
+            lambda: self._agent_runner.run(input),
         )
 
     def get_agent_runner(self) -> AgentRunner:

packages/sdk/server-ai/src/ldai/managed_model.py

@@ -45,8 +45,8 @@ async def invoke(self, prompt: str) -> ModelResponse:
         all_messages = config_messages + self._messages
 
         response = await tracker.track_metrics_of_async(
-            lambda: self._model_runner.invoke_model(all_messages),
             lambda result: result.metrics,
+            lambda: self._model_runner.invoke_model(all_messages),
         )
 
         input_text = '\r\n'.join(m.content for m in self._messages) if self._messages else ''

packages/sdk/server-ai/src/ldai/models.py

@@ -8,6 +8,31 @@
 from typing_extensions import Self
 
 
+@dataclass(frozen=True)
+class LDTool:
+    """
+    A single tool entry from the root-level tools map in an AI Config flag variation.
+    Distinct from model.parameters.tools[] which is the raw array passed to LLM providers.
+    """
+    name: str
+    description: Optional[str] = None
+    type: Optional[str] = None
+    parameters: Optional[Dict[str, Any]] = None
+    custom_parameters: Optional[Dict[str, Any]] = None
+
+    def to_dict(self) -> dict:
+        result: Dict[str, Any] = {'name': self.name}
+        if self.description is not None:
+            result['description'] = self.description
+        if self.type is not None:
+            result['type'] = self.type
+        if self.parameters is not None:
+            result['parameters'] = self.parameters
+        if self.custom_parameters is not None:
+            result['customParameters'] = self.custom_parameters  # camelCase in wire format
+        return result
+
+
 @dataclass
 class LDMessage:
     role: Literal['system', 'user', 'assistant']
@@ -211,6 +236,7 @@ class AICompletionConfigDefault(AIConfigDefault):
     """
     messages: Optional[List[LDMessage]] = None
     judge_configuration: Optional[JudgeConfiguration] = None
+    tools: Optional[Dict[str, 'LDTool']] = None
 
     def to_dict(self) -> dict:
         """
@@ -220,6 +246,8 @@ def to_dict(self) -> dict:
         result['messages'] = [message.to_dict() for message in self.messages] if self.messages else None
         if self.judge_configuration is not None:
             result['judgeConfiguration'] = self.judge_configuration.to_dict()
+        if self.tools is not None:
+            result['tools'] = {k: v.to_dict() for k, v in self.tools.items()}
         return result
 
 
@@ -231,6 +259,7 @@ class AICompletionConfig(AIConfig):
     evaluator: 'Evaluator' = field(kw_only=True, repr=False, compare=False, hash=False)
     messages: Optional[List[LDMessage]] = None
     judge_configuration: Optional[JudgeConfiguration] = None
+    tools: Optional[Dict[str, 'LDTool']] = None
 
     def to_dict(self) -> dict:
         """
@@ -240,6 +269,8 @@ def to_dict(self) -> dict:
         result['messages'] = [message.to_dict() for message in self.messages] if self.messages else None
         if self.judge_configuration is not None:
             result['judgeConfiguration'] = self.judge_configuration.to_dict()
+        if self.tools is not None:
+            result['tools'] = {k: v.to_dict() for k, v in self.tools.items()}
         return result
 
 
@@ -254,6 +285,7 @@ class AIAgentConfigDefault(AIConfigDefault):
     """
     instructions: Optional[str] = None
     judge_configuration: Optional[JudgeConfiguration] = None
+    tools: Optional[Dict[str, 'LDTool']] = None
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -264,6 +296,8 @@ def to_dict(self) -> Dict[str, Any]:
             result['instructions'] = self.instructions
         if self.judge_configuration is not None:
             result['judgeConfiguration'] = self.judge_configuration.to_dict()
+        if self.tools is not None:
+            result['tools'] = {k: v.to_dict() for k, v in self.tools.items()}
         return result
 
 
@@ -275,6 +309,7 @@ class AIAgentConfig(AIConfig):
     evaluator: 'Evaluator' = field(kw_only=True, repr=False, compare=False, hash=False)
     instructions: Optional[str] = None
     judge_configuration: Optional[JudgeConfiguration] = None
+    tools: Optional[Dict[str, 'LDTool']] = None
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -285,6 +320,8 @@ def to_dict(self) -> Dict[str, Any]:
             result['instructions'] = self.instructions
         if self.judge_configuration is not None:
             result['judgeConfiguration'] = self.judge_configuration.to_dict()
+        if self.tools is not None:
+            result['tools'] = {k: v.to_dict() for k, v in self.tools.items()}
         return result
 
 

packages/sdk/server-ai/src/ldai/tracker.py

@@ -262,8 +262,8 @@ def _track_from_metrics_extractor(
 
     def track_metrics_of(
         self,
-        func: Callable[[], Any],
         metrics_extractor: Callable[[Any], Any],
+        func: Callable[[], Any],
     ) -> Any:
         """
         Track metrics for a synchronous AI operation.
@@ -277,8 +277,8 @@ def track_metrics_of(
 
         For async operations, use :meth:`track_metrics_of_async`.
 
-        :param func: Synchronous callable that runs the operation
         :param metrics_extractor: Function that extracts LDAIMetrics from the operation result
+        :param func: Synchronous callable that runs the operation
         :return: The result of the operation
         """
         start_ns = time.perf_counter_ns()
@@ -294,14 +294,14 @@ def track_metrics_of(
         self.track_duration(duration)
         return self._track_from_metrics_extractor(result, metrics_extractor)
 
-    async def track_metrics_of_async(self, func, metrics_extractor):
+    async def track_metrics_of_async(self, metrics_extractor, func):
         """
         Track metrics for an async AI operation (``func`` is awaited).
 
         Same event semantics as :meth:`track_metrics_of`.
 
-        :param func: Async callable or zero-arg callable that returns an awaitable when called
         :param metrics_extractor: Function that extracts LDAIMetrics from the operation result
+        :param func: Async callable or zero-arg callable that returns an awaitable when called
         :return: The result of the operation
         """
         start_ns = time.perf_counter_ns()

packages/sdk/server-ai/tests/test_tools.py

@@ -0,0 +1,135 @@
+import pytest
+from ldclient import Config, Context, LDClient
+from ldclient.integrations.test_data import TestData
+
+from ldai import LDTool, LDAIClient
+from ldai.models import AIAgentConfigDefault, AICompletionConfigDefault
+
+
+@pytest.fixture
+def td() -> TestData:
+    td = TestData.data_source()
+    td.update(
+        td.flag('completion-with-tools')
+        .variations(
+            {
+                'model': {'name': 'gpt-5', 'parameters': {'temperature': 0.7}},
+                'messages': [{'role': 'user', 'content': 'Hello'}],
+                'tools': {
+                    'web-search-tool': {
+                        'name': 'web-search-tool',
+                        'type': 'function',
+                        'parameters': {'type': 'object', 'properties': {}, 'required': []},
+                        'customParameters': {'some-custom-parameter': 'some-custom-value'},
+                    }
+                },
+                '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1},
+            },
+        )
+        .variation_for_all(0)
+    )
+
+    td.update(
+        td.flag('completion-no-tools')
+        .variations(
+            {
+                'model': {'name': 'gpt-5'},
+                'messages': [{'role': 'user', 'content': 'Hello'}],
+                '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1},
+            },
+        )
+        .variation_for_all(0)
+    )
+
+    td.update(
+        td.flag('agent-with-tools')
+        .variations(
+            {
+                'model': {'name': 'gpt-5'},
+                'instructions': 'You are a helpful agent.',
+                'tools': {
+                    'search-tool': {
+                        'name': 'search-tool',
+                        'type': 'function',
+                        'customParameters': {'maxResults': 10},
+                    }
+                },
+                '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1, 'mode': 'agent'},
+            },
+        )
+        .variation_for_all(0)
+    )
+
+    return td
+
+
+@pytest.fixture
+def client(td) -> LDAIClient:
+    config = Config('fake-sdk-key', update_processor_class=td, send_events=False)
+    ld_client = LDClient(config=config)
+    return LDAIClient(ld_client)
+
+
+@pytest.fixture
+def context() -> Context:
+    return Context.builder('test-user').name('Test User').build()
+
+
+def test_completion_config_includes_tools_from_variation(client, context):
+    result = client.completion_config('completion-with-tools', context, AICompletionConfigDefault())
+
+    assert result.tools is not None
+    assert 'web-search-tool' in result.tools
+    tool = result.tools['web-search-tool']
+    assert tool.name == 'web-search-tool'
+    assert tool.type == 'function'
+    assert tool.custom_parameters == {'some-custom-parameter': 'some-custom-value'}
+
+
+def test_completion_config_tools_none_when_not_in_variation(client, context):
+    result = client.completion_config('completion-no-tools', context, AICompletionConfigDefault())
+
+    assert result.tools is None
+
+
+def test_completion_config_tools_none_when_variation_has_no_tools(client, context):
+    default_tool = LDTool(name='default-tool', type='function', custom_parameters={'priority': 'high'})
+    default = AICompletionConfigDefault(tools={'default-tool': default_tool})
+
+    result = client.completion_config('completion-no-tools', context, default)
+
+    assert result.tools is None
+
+
+def test_agent_config_includes_tools_from_variation(client, context):
+    result = client.agent_config('agent-with-tools', context, AIAgentConfigDefault())
+
+    assert result.tools is not None
+    assert 'search-tool' in result.tools
+    tool = result.tools['search-tool']
+    assert tool.name == 'search-tool'
+    assert tool.custom_parameters == {'maxResults': 10}
+
+
+def test_aitool_to_dict_serializes_custom_parameters_as_camel_case():
+    tool = LDTool(
+        name='my-tool',
+        type='function',
+        parameters={'type': 'object'},
+        custom_parameters={'someKey': 'someValue'},
+    )
+    d = tool.to_dict()
+
+    assert d['name'] == 'my-tool'
+    assert d['type'] == 'function'
+    assert d['parameters'] == {'type': 'object'}
+    assert 'customParameters' in d
+    assert d['customParameters'] == {'someKey': 'someValue'}
+    assert 'custom_parameters' not in d
+
+
+def test_aitool_to_dict_omits_none_fields():
+    tool = LDTool(name='bare-tool')
+    d = tool.to_dict()
+
+    assert d == {'name': 'bare-tool'}