fix openai tool call tracking and lint issues

Author: jsonbailey

packages/ai-providers/server-ai-langchain/src/ldai_langchain/langchain_agent_runner.py

@@ -6,7 +6,10 @@
 from ldai.providers import AgentResult, AgentRunner
 from ldai.providers.types import LDAIMetrics
 
-from ldai_langchain.langchain_helper import extract_last_message_content, sum_token_usage_from_messages
+from ldai_langchain.langchain_helper import (
+    extract_last_message_content,
+    sum_token_usage_from_messages,
+)
 
 
 class LangChainAgentRunner(AgentRunner):

packages/ai-providers/server-ai-langchain/src/ldai_langchain/langchain_helper.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from langchain_core.language_models.chat_models import BaseChatModel
 from langchain_core.messages import AIMessage, HumanMessage, SystemMessage

packages/ai-providers/server-ai-openai/src/ldai_openai/openai_agent_graph_runner.py

@@ -185,7 +185,7 @@ def build_node(node: AgentGraphNode, ctx: dict) -> Any:
                 # Map runtime tool name → LD config key for metrics (function __name__
                 # for callables; identity for native tool instances — see get_tool_calls_from_run_items).
                 if is_agent_tool_instance(tool_fn):
-                    tool_name_map[f'{tool_fn.name}_call'] = tool_name
+                    tool_name_map[tool_fn.name] = tool_name
                 else:
                     tool_name_map[tool_fn.__name__] = tool_name
                 agent_tools.append(registry_value_to_agent_tool(tool_fn))
@@ -290,10 +290,11 @@ def _track_tool_calls(self, result: Any, tracker: Any) -> None:
         """Track all tool calls from the run result, attributed to the node that called them."""
         gk = tracker.graph_key if tracker is not None else None
         for agent_name, tool_fn_name in get_tool_calls_from_run_items(result.new_items):
-            log.info(f"Tracking tool call: agent_name={agent_name}, tool_fn_name={tool_fn_name}")
-            original_key = self._agent_name_map.get(agent_name, agent_name)
-            tool_name = self._tool_name_map.get(tool_fn_name, '')
-            node = self._graph.get_node(original_key)
+            agent_key = self._agent_name_map.get(agent_name, agent_name)
+            tool_name = self._tool_name_map.get(tool_fn_name)
+            if tool_name is None:
+                continue
+            node = self._graph.get_node(agent_key)
             if node is None:
                 continue
             config_tracker = node.get_config().tracker

packages/ai-providers/server-ai-openai/src/ldai_openai/openai_agent_runner.py

@@ -6,7 +6,10 @@
 from ldai.providers import AgentResult, AgentRunner, ToolRegistry
 from ldai.providers.types import LDAIMetrics
 
-from ldai_openai.openai_helper import get_ai_usage_from_response, registry_value_to_agent_tool
+from ldai_openai.openai_helper import (
+    get_ai_usage_from_response,
+    registry_value_to_agent_tool,
+)
 
 
 class OpenAIAgentRunner(AgentRunner):

packages/ai-providers/server-ai-openai/src/ldai_openai/openai_helper.py

@@ -1,4 +1,3 @@
-import typing
 from typing import Any, Dict, Iterable, List, Optional, Tuple, cast
 
 from ldai import LDMessage
@@ -7,7 +6,6 @@
 from openai.types.chat import ChatCompletionMessageParam
 
 
-
 def convert_messages_to_openai(messages: List[LDMessage]) -> Iterable[ChatCompletionMessageParam]:
     """
     Convert LaunchDarkly messages to OpenAI chat completion message format.
@@ -109,43 +107,9 @@ def normalize_tool_types(tool_definitions: List[Any]) -> List[Dict[str, Any]]:
     return result
 
 
-# Native tool raw_item type names don't always match the LD config key convention.
-_NATIVE_TOOL_TYPE_TO_CONFIG_KEY = {
-    'web_search': 'web_search_tool',
-}
-
-# ``agents.Tool`` is a typing.Union of concrete tool classes, not a runtime class.
-# Using ``isinstance(x, Tool)`` raises TypeError (subscripted generics / union checks).
-_AGENT_TOOL_TYPES: Optional[Tuple[type, ...]] = None
-
-
-def _concrete_agent_tool_types() -> Tuple[type, ...]:
-    """Resolve concrete classes behind ``agents.Tool`` (a Union alias)."""
-    try:
-        from agents import Tool as ToolUnion
-    except ImportError:
-        return ()
-    args = typing.get_args(ToolUnion)
-    if not args:
-        return ()
-    out: List[type] = []
-    for a in args:
-        origin = getattr(a, '__origin__', None)
-        if origin is not None and isinstance(origin, type):
-            out.append(origin)
-        elif isinstance(a, type):
-            out.append(a)
-    return tuple(out)
-
-
 def is_agent_tool_instance(value: Any) -> bool:
     """True if ``value`` is already an openai-agents tool object (not a plain callable)."""
-    global _AGENT_TOOL_TYPES
-    if _AGENT_TOOL_TYPES is None:
-        _AGENT_TOOL_TYPES = _concrete_agent_tool_types()
-    if not _AGENT_TOOL_TYPES:
-        return False
-    return isinstance(value, _AGENT_TOOL_TYPES)
+    return not callable(value)
 
 
 def registry_value_to_agent_tool(value: Any) -> Any:
@@ -156,19 +120,27 @@ def registry_value_to_agent_tool(value: Any) -> Any:
     tool instances (e.g. ``WebSearchTool()``, ``FileSearchTool(...)``) are
     returned unchanged so they are not double-wrapped.
     """
+    if is_agent_tool_instance(value):
+        return value
     try:
         from agents import function_tool
     except ImportError as exc:
         raise ImportError(
             "openai-agents is required for agent tools. "
             "Install it with: pip install openai-agents"
         ) from exc
-
-    if is_agent_tool_instance(value):
-        return value
     return function_tool(value)
 
 
+# Native tool response types do not match the SDK or LD tool name; this map aligns them.
+# Function tools are omitted—they already arrive as ``ResponseFunctionToolCall.name``.
+_RESPONSE_TYPE_TO_TOOL_NAME: Dict[str, str] = {
+    'web_search_call': 'web_search',
+    'file_search_call': 'file_search',
+    'code_interpreter_call': 'code_interpreter',
+}
+
+
 def get_tool_calls_from_run_items(new_items: List[Any]) -> List[Tuple[str, str]]:
     """
     Extract (agent_name, tool_name) pairs from RunResult.new_items.
@@ -197,9 +169,9 @@ def get_tool_calls_from_run_items(new_items: List[Any]) -> List[Tuple[str, str]]
             tool_name = raw.name
         else:
             raw_type = getattr(raw, 'type', None) or (raw.get('type') if isinstance(raw, dict) else None)
-            if not raw_type:
+            if not isinstance(raw_type, str):
                 continue
-            tool_name = _NATIVE_TOOL_TYPE_TO_CONFIG_KEY.get(raw_type, raw_type)
+            tool_name = _RESPONSE_TYPE_TO_TOOL_NAME.get(raw_type, raw_type)
         if tool_name:
             result.append((agent_name, tool_name))
     return result

packages/ai-providers/server-ai-openai/tests/test_tracking_openai_agents.py

@@ -109,6 +109,19 @@ def _make_run_result(
     return result
 
 
+def _tool_registry(*config_names: str) -> dict:
+    """Registry entries whose callable __name__ matches runtime tool names from the SDK."""
+
+    def _stub(name: str):
+        def fn():
+            pass
+
+        fn.__name__ = name
+        return fn
+
+    return {n: _stub(n) for n in config_names}
+
+
 def _make_tool_call_item(agent_name: str, tool_name: str) -> MagicMock:
     """
     Create a mock ToolCallItem with a ResponseFunctionToolCall raw item so that
@@ -300,15 +313,15 @@ async def test_tracks_graph_key_on_node_events():
 
 @pytest.mark.asyncio
 async def test_tracks_tool_calls_from_run_items():
-    """A tool_call event fires for each tool found in RunResult.new_items."""
+    """A tool_call event fires for tools registered on the graph and in the tool registry."""
     mock_ld_client = MagicMock()
-    graph = _make_graph(mock_ld_client, node_key='root-agent')
+    graph = _make_graph(mock_ld_client, node_key='root-agent', tool_names=['get_weather'])
 
     tool_item = _make_tool_call_item('root-agent', 'get_weather')
     run_result = _make_run_result(output='done', tool_call_items=[tool_item])
 
     with patch.dict('sys.modules', _make_agents_modules(run_result)):
-        runner = OpenAIAgentGraphRunner(graph, {})
+        runner = OpenAIAgentGraphRunner(graph, _tool_registry('get_weather'))
         await runner.run('What is the weather?')
 
     ev = _events(mock_ld_client)
@@ -319,9 +332,11 @@ async def test_tracks_tool_calls_from_run_items():
 
 @pytest.mark.asyncio
 async def test_tracks_multiple_tool_calls():
-    """One tool_call event fires per tool in RunResult.new_items."""
+    """One tool_call event fires per registered tool in RunResult.new_items."""
     mock_ld_client = MagicMock()
-    graph = _make_graph(mock_ld_client, node_key='root-agent')
+    graph = _make_graph(
+        mock_ld_client, node_key='root-agent', tool_names=['search', 'summarize']
+    )
 
     items = [
         _make_tool_call_item('root-agent', 'search'),
@@ -330,14 +345,31 @@ async def test_tracks_multiple_tool_calls():
     run_result = _make_run_result(output='done', tool_call_items=items)
 
     with patch.dict('sys.modules', _make_agents_modules(run_result)):
-        runner = OpenAIAgentGraphRunner(graph, {})
+        runner = OpenAIAgentGraphRunner(graph, _tool_registry('search', 'summarize'))
         await runner.run('Search and summarize.')
 
     ev = _events(mock_ld_client)
     tool_keys = [data['toolKey'] for data, _ in ev.get('$ld:ai:tool_call', [])]
     assert sorted(tool_keys) == ['search', 'summarize']
 
 
+@pytest.mark.asyncio
+async def test_does_not_track_tool_calls_without_graph_and_registry_config():
+    """RunResult tool items that are not backed by graph + registry tools are ignored."""
+    mock_ld_client = MagicMock()
+    graph = _make_graph(mock_ld_client, node_key='root-agent')
+
+    tool_item = _make_tool_call_item('root-agent', 'orphan_tool')
+    run_result = _make_run_result(output='done', tool_call_items=[tool_item])
+
+    with patch.dict('sys.modules', _make_agents_modules(run_result)):
+        runner = OpenAIAgentGraphRunner(graph, {})
+        await runner.run('prompt')
+
+    ev = _events(mock_ld_client)
+    assert ev.get('$ld:ai:tool_call', []) == []
+
+
 @pytest.mark.asyncio
 async def test_tracks_failure_and_latency_on_runner_error():
     """When Runner.run raises, failure and latency events fire; success does not."""

packages/sdk/server-ai/src/ldai/providers/runner_factory.py

@@ -3,9 +3,9 @@
 
 from ldai import log
 from ldai.models import AIConfigKind
-from ldai.providers.ai_provider import AIProvider
 from ldai.providers.agent_graph_runner import AgentGraphRunner
 from ldai.providers.agent_runner import AgentRunner
+from ldai.providers.ai_provider import AIProvider
 from ldai.providers.model_runner import ModelRunner
 
 T = TypeVar('T')