Refactor ToolFailureEvaluator: privatize models, rename attempts to failed_attempts

Signed-off-by: Eric Evans <194135482+ericevans-nv@users.noreply.github.com>

Author: ericevans-nv

packages/nvidia_nat_core/tests/nat/utils/test_atif_converter.py

@@ -708,11 +708,6 @@ def test_stream_matches_batch(
                 assert len(s_step.tool_calls) == len(b_step.tool_calls)
 
 
-# ---------------------------------------------------------------------------
-# Tool error → ATIF conversion tests
-# ---------------------------------------------------------------------------
-
-
 @pytest.fixture(name="error_trajectory")
 def fixture_error_trajectory() -> list[IntermediateStep]:
     """Trajectory with one successful and one failed tool call."""

packages/nvidia_nat_eval/src/nat/plugins/eval/tool_failure_evaluator/__init__.py

@@ -14,13 +14,9 @@
 # limitations under the License.
 
 from .evaluator import ToolFailureEvaluator
-from .models import ToolFailureReasoning
-from .models import ToolSummary
 from .register import ToolFailureEvaluatorConfig
 
 __all__ = [
     "ToolFailureEvaluator",
     "ToolFailureEvaluatorConfig",
-    "ToolFailureReasoning",
-    "ToolSummary",
 ]

packages/nvidia_nat_eval/src/nat/plugins/eval/tool_failure_evaluator/evaluator.py

@@ -29,9 +29,9 @@
 from nat.plugins.eval.evaluator.atif_evaluator import AtifEvalSample
 from nat.plugins.eval.evaluator.base_evaluator import BaseEvaluator
 
-from .models import ToolFailureReasoning
-from .models import ToolSummary
 from .models import _ToolCall
+from .models import _ToolFailureReasoning
+from .models import _ToolSummary
 
 
 class ToolFailureEvaluator(BaseEvaluator, AtifBaseEvaluator):
@@ -47,7 +47,7 @@ def __init__(self, max_concurrency: int = 8):
     async def evaluate_item(self, item: EvalInputItem) -> EvalOutputItem:
         """Evaluate a single item's legacy trajectory for tool failures."""
         if not item.trajectory:
-            return EvalOutputItem(id=item.id, score=1.0, reasoning=ToolFailureReasoning())
+            return EvalOutputItem(id=item.id, score=1.0, reasoning=_ToolFailureReasoning())
 
         total_tool_calls: int = 0
         failed_tool_calls: int = 0
@@ -71,17 +71,17 @@ async def evaluate_item(self, item: EvalInputItem) -> EvalOutputItem:
                 failed_tool_calls += 1
 
         score: float = self._success_rate(total_tool_calls, failed_tool_calls)
-        per_tool_summary: list[ToolSummary] = [
-            ToolSummary(
+        per_tool_summary: list[_ToolSummary] = [
+            _ToolSummary(
                 tool_name=name,
                 total_calls=len(attempts),
                 failed_calls=failed_count,
-                attempts=[a for a in attempts if a.error is not None],
+                failed_attempts=[a for a in attempts if a.error is not None],
             ) for name, attempts in calls_by_tool.items()
             if (failed_count := sum(1 for a in attempts if a.error is not None)) > 0
         ]
         failed_tools: list[str] = [ts.tool_name for ts in per_tool_summary]
-        reasoning: ToolFailureReasoning = ToolFailureReasoning(
+        reasoning: _ToolFailureReasoning = _ToolFailureReasoning(
             total_tool_calls=total_tool_calls,
             failed_tool_calls=failed_tool_calls,
             failed_tools=failed_tools,
@@ -154,17 +154,17 @@ async def evaluate_atif_item(self, sample: AtifEvalSample) -> EvalOutputItem:
                     failed_tool_calls += 1
 
         score: float = self._success_rate(total_tool_calls, failed_tool_calls)
-        per_tool_summary: list[ToolSummary] = [
-            ToolSummary(
+        per_tool_summary: list[_ToolSummary] = [
+            _ToolSummary(
                 tool_name=name,
                 total_calls=len(attempts),
                 failed_calls=failed_count,
-                attempts=[a for a in attempts if a.error is not None],
+                failed_attempts=[a for a in attempts if a.error is not None],
             ) for name, attempts in calls_by_tool.items()
             if (failed_count := sum(1 for a in attempts if a.error is not None)) > 0
         ]
         failed_tools: list[str] = [ts.tool_name for ts in per_tool_summary]
-        reasoning: ToolFailureReasoning = ToolFailureReasoning(
+        reasoning: _ToolFailureReasoning = _ToolFailureReasoning(
             total_tool_calls=total_tool_calls,
             failed_tool_calls=failed_tool_calls,
             failed_tools=failed_tools,

packages/nvidia_nat_eval/src/nat/plugins/eval/tool_failure_evaluator/models.py

@@ -30,26 +30,26 @@ class _ToolCall(BaseModel):
     error: str | None = Field(default=None, description="Error string if failed, None if succeeded.")
 
 
-class ToolSummary(BaseModel):
+class _ToolSummary(BaseModel):
     """Complete health and attempt data for a single tool."""
 
     tool_name: str = Field(description="Name of the tool.")
     total_calls: int = Field(default=0, description="Total number of calls to this tool.")
     failed_calls: int = Field(default=0, description="Number of calls that returned an error.")
-    attempts: list[_ToolCall] = Field(
+    failed_attempts: list[_ToolCall] = Field(
         default_factory=list,
-        description="Ordered list of every call to this tool.",
+        description="Ordered list of failed calls to this tool.",
     )
 
 
-class ToolFailureReasoning(BaseModel):
+class _ToolFailureReasoning(BaseModel):
     """Complete reasoning payload returned by the tool failure evaluator."""
 
     total_tool_calls: int = Field(default=0, description="Total tool calls in the trajectory.")
     failed_tool_calls: int = Field(default=0, description="Total tool calls that errored.")
     failed_tools: list[str] = Field(default_factory=list, description="Names of tools that had at least one failure.")
     score: float = Field(default=1.0, description="Overall success rate (0.0-1.0).")
-    per_tool_summary: list[ToolSummary] = Field(
+    per_tool_summary: list[_ToolSummary] = Field(
         default_factory=list,
         description="Per-tool health summary with attempt history.",
     )

packages/nvidia_nat_eval/tests/eval/evaluator/test_tool_failure_evaluator.py

@@ -12,12 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Unit tests for ToolFailureEvaluator model population.
-
-Validates that ToolFailureReasoning, ToolSummary, and _ToolCall are correctly
-populated from both the legacy IntermediateStep lane and the ATIF lane, and
-that error detection correctly distinguishes failures from successes.
-"""
+"""Unit tests for ToolFailureEvaluator."""
 
 from __future__ import annotations
 
@@ -39,7 +34,7 @@
 from nat.data_models.invocation_node import InvocationNode
 from nat.plugins.eval.evaluator.atif_evaluator import AtifEvalSample
 from nat.plugins.eval.tool_failure_evaluator.evaluator import ToolFailureEvaluator
-from nat.plugins.eval.tool_failure_evaluator.models import ToolFailureReasoning
+from nat.plugins.eval.tool_failure_evaluator.models import _ToolFailureReasoning
 
 _DUMMY_ANCESTRY: InvocationNode = InvocationNode(function_id="f-0", function_name="test_fn")
 
@@ -107,18 +102,16 @@ def evaluator_fixture() -> ToolFailureEvaluator:
     return ToolFailureEvaluator()
 
 
-class TestLegacyLaneModelPopulation:
-    """Verify ToolFailureReasoning, ToolSummary, and _ToolCall are correctly
-    populated from legacy IntermediateStep trajectories.
-    """
+class TestEvaluateIntermediateStepTrajectory:
+    """Tests for evaluating IntermediateStep trajectories."""
 
     async def test_empty_trajectory_produces_default_reasoning(self, evaluator: ToolFailureEvaluator):
         """An empty trajectory should yield default ToolFailureReasoning with
         zero counts, no failed tools, and a perfect score.
         """
         result = await evaluator.evaluate_item(_eval_input("empty", []))
 
-        reasoning: ToolFailureReasoning = result.reasoning
+        reasoning: _ToolFailureReasoning = result.reasoning
         assert reasoning.total_tool_calls == 0
         assert reasoning.failed_tool_calls == 0
         assert reasoning.failed_tools == []
@@ -136,7 +129,7 @@ async def test_all_failed_calls_populate_summary_with_error_details(self, evalua
         ]
         result = await evaluator.evaluate_item(_eval_input("fail", trajectory))
 
-        reasoning: ToolFailureReasoning = result.reasoning
+        reasoning: _ToolFailureReasoning = result.reasoning
         assert reasoning.total_tool_calls == 2
         assert reasoning.failed_tool_calls == 2
         assert reasoning.failed_tools == ["lookup"]
@@ -146,8 +139,8 @@ async def test_all_failed_calls_populate_summary_with_error_details(self, evalua
         assert summary.tool_name == "lookup"
         assert summary.total_calls == 2
         assert summary.failed_calls == 2
-        assert len(summary.attempts) == 2
-        for attempt in summary.attempts:
+        assert len(summary.failed_attempts) == 2
+        for attempt in summary.failed_attempts:
             assert attempt.error == "ValueError: bad input"
             assert attempt.output is None
 
@@ -163,7 +156,7 @@ async def test_mixed_results_split_correctly_across_models(self, evaluator: Tool
         ]
         result = await evaluator.evaluate_item(_eval_input("mixed", trajectory))
 
-        reasoning: ToolFailureReasoning = result.reasoning
+        reasoning: _ToolFailureReasoning = result.reasoning
         assert reasoning.total_tool_calls == 2
         assert reasoning.failed_tool_calls == 1
         assert reasoning.failed_tools == ["lookup"]
@@ -173,7 +166,7 @@ async def test_mixed_results_split_correctly_across_models(self, evaluator: Tool
         assert reasoning.per_tool_summary[0].tool_name == "lookup"
 
     async def test_same_tool_mixed_results_filters_attempts_to_failures_only(self, evaluator: ToolFailureEvaluator):
-        """When a single tool has both successes and failures, ToolSummary.attempts
+        """When a single tool has both successes and failures, ToolSummary.failed_attempts
         should contain only the failed _ToolCall entries while total_calls reflects all.
         """
         trajectory = [
@@ -184,13 +177,13 @@ async def test_same_tool_mixed_results_filters_attempts_to_failures_only(self, e
         ]
         result = await evaluator.evaluate_item(_eval_input("filter", trajectory))
 
-        reasoning: ToolFailureReasoning = result.reasoning
+        reasoning: _ToolFailureReasoning = result.reasoning
         summary = reasoning.per_tool_summary[0]
         assert summary.total_calls == 2
         assert summary.failed_calls == 1
-        assert len(summary.attempts) == 1
-        assert summary.attempts[0].error == "boom"
-        assert summary.attempts[0].input == {"q": "bad"}
+        assert len(summary.failed_attempts) == 1
+        assert summary.failed_attempts[0].error == "boom"
+        assert summary.failed_attempts[0].input == {"q": "bad"}
 
     async def test_none_data_on_step_is_not_treated_as_error(self, evaluator: ToolFailureEvaluator):
         """A TOOL_END step with data=None should count as a call but not a failure."""
@@ -220,10 +213,8 @@ async def test_missing_tool_name_recorded_as_unknown(self, evaluator: ToolFailur
         assert result.reasoning.per_tool_summary[0].tool_name == "unknown"
 
 
-class TestAtifLaneModelPopulation:
-    """Verify ToolFailureReasoning, ToolSummary, and _ToolCall are correctly
-    populated from ATIF trajectories using each error detection path.
-    """
+class TestEvaluateAtifTrajectory:
+    """Tests for evaluating ATIF trajectories."""
 
     async def test_error_detected_via_extra_tool_errors(self, evaluator: ToolFailureEvaluator):
         """Structured error metadata in step.extra['tool_errors'] should populate
@@ -242,11 +233,11 @@ async def test_error_detected_via_extra_tool_errors(self, evaluator: ToolFailure
         ]
         result = await evaluator.evaluate_atif_item(_atif_sample("extra", steps))
 
-        reasoning: ToolFailureReasoning = result.reasoning
+        reasoning: _ToolFailureReasoning = result.reasoning
         assert reasoning.failed_tool_calls == 1
         assert reasoning.failed_tools == ["lookup"]
-        assert reasoning.per_tool_summary[0].attempts[0].error == "ValueError: Column not found"
-        assert reasoning.per_tool_summary[0].attempts[0].input == {"query": "q1"}
+        assert reasoning.per_tool_summary[0].failed_attempts[0].error == "ValueError: Column not found"
+        assert reasoning.per_tool_summary[0].failed_attempts[0].input == {"query": "q1"}
 
     async def test_error_detected_via_stringified_tool_message_dict(self, evaluator: ToolFailureEvaluator):
         """A Python dict literal with status='error' in the observation content
@@ -262,7 +253,7 @@ async def test_error_detected_via_stringified_tool_message_dict(self, evaluator:
         result = await evaluator.evaluate_atif_item(_atif_sample("parsed", steps))
 
         assert result.reasoning.failed_tool_calls == 1
-        assert result.reasoning.per_tool_summary[0].attempts[0].error == "TimeoutError: timed out"
+        assert result.reasoning.per_tool_summary[0].failed_attempts[0].error == "TimeoutError: timed out"
 
     async def test_error_detected_via_raw_error_pattern(self, evaluator: ToolFailureEvaluator):
         """Observation content matching 'XyzError: ...' should be detected as a
@@ -274,7 +265,7 @@ async def test_error_detected_via_raw_error_pattern(self, evaluator: ToolFailure
         result = await evaluator.evaluate_atif_item(_atif_sample("pattern", steps))
 
         assert result.reasoning.failed_tool_calls == 1
-        assert result.reasoning.per_tool_summary[0].attempts[0].error == "RuntimeError: internal failure"
+        assert result.reasoning.per_tool_summary[0].failed_attempts[0].error == "RuntimeError: internal failure"
 
     async def test_extra_tool_errors_takes_priority_over_observation_pattern(self, evaluator: ToolFailureEvaluator):
         """When both extra['tool_errors'] and a raw error pattern match, the
@@ -292,7 +283,7 @@ async def test_extra_tool_errors_takes_priority_over_observation_pattern(self, e
         ]
         result = await evaluator.evaluate_atif_item(_atif_sample("priority", steps))
 
-        assert result.reasoning.per_tool_summary[0].attempts[0].error == "ValueError: from extra"
+        assert result.reasoning.per_tool_summary[0].failed_attempts[0].error == "ValueError: from extra"
 
     async def test_mixed_success_and_failure_populates_only_failing_tool(self, evaluator: ToolFailureEvaluator):
         """With one successful and one failing tool, only the failing tool

packages/nvidia_nat_langchain/src/nat/plugins/langchain/callback_handler.py

@@ -36,6 +36,8 @@
 from nat.data_models.intermediate_step import IntermediateStepType
 from nat.data_models.intermediate_step import ServerToolUseSchema
 from nat.data_models.intermediate_step import StreamEventData
+from nat.data_models.intermediate_step import ToolDetails
+from nat.data_models.intermediate_step import ToolParameters
 from nat.data_models.intermediate_step import ToolSchema
 from nat.data_models.intermediate_step import TraceMetadata
 from nat.data_models.intermediate_step import UsageInfo
@@ -53,11 +55,28 @@ def _extract_tools_schema(invocation_params: dict) -> list:
             try:
                 tools_schema.append(ToolSchema(**tool))
             except Exception:
-                logger.debug(
-                    "Failed to parse tool schema from invocation params: %s. \n This "
-                    "can occur when the LLM server has native tools and can be ignored if "
-                    "using the responses API.",
-                    tool)
+                # Handle non-OpenAI tool formats (e.g. Anthropic: top-level name/description/input_schema)
+                try:
+                    input_schema = tool.get("input_schema") or {}
+                    tools_schema.append(
+                        ToolSchema(
+                            type="function",
+                            function=ToolDetails(
+                                name=tool["name"],
+                                description=tool.get("description", ""),
+                                parameters=ToolParameters(
+                                    properties=input_schema.get("properties", {}),
+                                    required=input_schema.get("required", []),
+                                    additionalProperties=input_schema.get("additionalProperties", False),
+                                ),
+                            ),
+                        ))
+                except (KeyError, TypeError, AttributeError):
+                    logger.debug(
+                        "Failed to parse tool schema from invocation params: %s. \n This "
+                        "can occur when the LLM server has native tools and can be ignored if "
+                        "using the responses API.",
+                        tool)
 
     return tools_schema
 

packages/nvidia_nat_langchain/tests/test_langchain_callback_handler.py

@@ -284,11 +284,6 @@ def test_extract_tools_schema_empty_and_none():
     assert _extract_tools_schema(None) == []
 
 
-# ---------------------------------------------------------------------------
-# on_tool_error tests
-# ---------------------------------------------------------------------------
-
-
 @pytest.fixture(name="handler_and_stats")
 def fixture_handler_and_stats(
     reactive_stream: Subject, ) -> tuple[LangchainProfilerHandler, list[IntermediateStepPayload]]: