feat(evaluator): add ATIF trace and log read handles for evidence

Add the ATIF (Agent Trace Interchange Format) model and normalizers for NAT
trajectories, OpenTelemetry GenAI spans, and OpenInference spans, plus the
TraceHandle and LogHandle read handles exposed via CandidateEvidence.trace()
and CandidateEvidence.logs().

Includes the example-only inefficient_retry_loop metric that scores over the
normalized trace evidence.

Vendored mirror synced via make vendor.

Signed-off-by: Arpit Singh (SW-CLOUD) <arpsingh@nvidia.com>

Author: arpitsardhana

packages/nemo_evaluator_sdk/examples/run_agent_eval/example_metrics.py

@@ -3,19 +3,22 @@
 
 """Reference metrics-over-evidence for this example (not SDK API).
 
-These show how to score from the SDK's filesystem evidence handle instead of a
-stamped verifier reward:
+These show how to score from the SDK's evidence handles instead of a stamped
+verifier reward:
 
 * :class:`TestsPassMetric` runs a command against ``final_state`` filesystem
   evidence (in a throwaway overlay) and scores on exit 0.
 * :class:`NoTestCheatingMetric` diffs ``initial_state`` against ``final_state``
   and fails if the agent touched protected (e.g. test) paths.
+* :class:`InefficientRetryLoopMetric` reads the normalized ``trace`` and fails
+  when the same tool call repeats past a threshold.
 """
 
 from __future__ import annotations
 
 from collections.abc import Sequence
 
+from nemo_evaluator_sdk.agent_eval.trials import EVIDENCE_FINAL_STATE, EVIDENCE_INITIAL_STATE, EVIDENCE_TRACE
 from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
 
 
@@ -26,7 +29,7 @@ def __init__(
         self,
         command: Sequence[str],
         *,
-        evidence_name: str = "final_state",
+        evidence_name: str = EVIDENCE_FINAL_STATE,
         cwd: str = ".",
         timeout_s: float = 300.0,
     ) -> None:
@@ -60,8 +63,8 @@ def __init__(
         *,
         protected: Sequence[str] = ("tests/",),
         change_types: Sequence[str] = ("added", "modified", "deleted"),
-        initial_name: str = "initial_state",
-        final_name: str = "final_state",
+        initial_name: str = EVIDENCE_INITIAL_STATE,
+        final_name: str = EVIDENCE_FINAL_STATE,
     ) -> None:
         self._protected = tuple(protected)
         self._change_types = set(change_types)
@@ -87,3 +90,38 @@ async def compute_scores(self, input: MetricInput) -> MetricResult:
             ]
             clean = not violations
         return MetricResult(outputs=[MetricOutput(name="no_test_cheating", value=clean)])
+
+
+class InefficientRetryLoopMetric:
+    """Score ``False`` when the same tool call repeats more than ``threshold`` times."""
+
+    def __init__(self, *, threshold: int = 2, evidence_name: str = EVIDENCE_TRACE) -> None:
+        self._threshold = threshold
+        self._evidence_name = evidence_name
+
+    @property
+    def type(self) -> str:
+        return "inefficient_retry_loop"
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        return [
+            MetricOutputSpec.boolean("efficient_tool_use"),
+            MetricOutputSpec.discrete_score("max_repeated_tool_calls"),
+        ]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        max_repeats = 0
+        evidence = input.candidate.evidence
+        if evidence is not None and evidence.get(self._evidence_name) is not None:
+            calls = await (await evidence.trace(self._evidence_name)).tool_calls()
+            counts: dict[str, int] = {}
+            for call in calls:
+                key = f"{call.function_name}:{sorted((call.arguments or {}).items())}"
+                counts[key] = counts.get(key, 0) + 1
+            max_repeats = max(counts.values(), default=0)
+        return MetricResult(
+            outputs=[
+                MetricOutput(name="efficient_tool_use", value=max_repeats <= self._threshold),
+                MetricOutput(name="max_repeated_tool_calls", value=max_repeats),
+            ]
+        )

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py

@@ -44,7 +44,7 @@
 from nemo_evaluator_sdk.metrics.protocol import Metric, validate_metric_result
 from nemo_evaluator_sdk.metrics.utils import metric_type_name
 from nemo_evaluator_sdk.values import Agent, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel
-from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
+from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor, normalize_trace_descriptor
 from openai import AsyncOpenAI
 
 log = getLogger(__name__)
@@ -327,7 +327,9 @@ def _trial_from_sample(task: AgentEvalTask, target: Model | Agent, sample: dict[
         # trial stays scorable instead of being dropped as empty output.
         output_text = _reasoning_content_fallback(sample.get("response"))
     if "trajectory" in sample:
-        trace = EvidenceDescriptor(kind="trace", format="json", data=sample["trajectory"])
+        # Normalize to ATIF before the trial is persisted so the stored shape is
+        # source-agnostic (sources in, ATIF out); TraceHandle then reads it uniformly.
+        trace = normalize_trace_descriptor(EvidenceDescriptor(kind="trace", format="json", data=sample["trajectory"]))
     else:
         trace = EvidenceDescriptor(kind="sdk_online_generation", data={"task_id": task.id, "target": target.name})
 

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py

@@ -14,7 +14,7 @@
 
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.values import Agent, Model
-from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
+from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor, normalize_trace_descriptor
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 # Well-known evidence keys produced by ``standard_evidence_descriptors``. Harness
@@ -158,10 +158,14 @@ def standard_evidence_descriptors(
     if trace_path is not None:
         trace_name = Path(trace_path).name.lower()
         is_atif = trace_name.startswith("atif") or ".atif." in trace_name
-        descriptors[EVIDENCE_TRACE] = EvidenceDescriptor(
-            kind="trace",
-            format="atif" if is_atif else "json",
-            ref=str(trace_path),
+        # Normalize the source trace into a sibling ATIF file before persistence so the
+        # stored descriptor is ATIF regardless of producer (no-op if already ATIF/missing).
+        descriptors[EVIDENCE_TRACE] = normalize_trace_descriptor(
+            EvidenceDescriptor(
+                kind="trace",
+                format="atif" if is_atif else "json",
+                ref=str(trace_path),
+            )
         )
 
     logs_metadata = {"primary_log": primary_log} if primary_log else {}

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py

@@ -4,6 +4,15 @@
 """Public value types for evaluator SDK runtime."""
 
 from nemo_evaluator_sdk.values.agents import Agent
+from nemo_evaluator_sdk.values.atif import (
+    FinalMetrics,
+    Metrics,
+    Observation,
+    ObservationResult,
+    Step,
+    ToolCall,
+    Trajectory,
+)
 from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes
 from nemo_evaluator_sdk.values.dataset_schemas import (
     FieldMapping,
@@ -17,6 +26,11 @@
     FilesystemDiff,
     FilesystemEntry,
     LocalFilesystemEvidence,
+    LogHandle,
+    TraceHandle,
+    WellKnownEvidenceKey,
+    normalize_candidate_evidence,
+    normalize_trace_descriptor,
 )
 from nemo_evaluator_sdk.values.metrics import (
     BLEU,
@@ -109,6 +123,18 @@
     "ContinuousScore",
     "FilesystemDiff",
     "FilesystemEntry",
+    "FinalMetrics",
+    "LogHandle",
+    "Metrics",
+    "Observation",
+    "ObservationResult",
+    "Step",
+    "ToolCall",
+    "Trajectory",
+    "TraceHandle",
+    "WellKnownEvidenceKey",
+    "normalize_candidate_evidence",
+    "normalize_trace_descriptor",
     "DatasetRow",
     "DatasetRows",
     "DefaultAggregateFieldName",