fix(tracing): fail loud on span_name in Fireworks adapter

The /v1/traces endpoint returns flat PascalCase rows (one LLM call each) —
there is no nested span/generation structure to walk. The leftover
snake_case observations branch in extract_messages_from_trace_dict and
the entire get_final_generation_in_span_dict helper would always return
no messages, so any caller passing span_name silently got an empty
EvaluationRow (returning None).

Raise NotImplementedError instead:
- get_evaluation_rows rejects span_name upfront unless a custom converter
  is supplied (skips the pointless HTTP round-trip)
- convert_trace_dict_to_evaluation_row and extract_messages_from_trace_dict
  also raise, for defense in depth
- delete the dead get_final_generation_in_span_dict helper

Made-with: Cursor

Author: xzrderek

eval_protocol/adapters/fireworks_tracing.py

@@ -56,11 +56,24 @@ def convert_trace_dict_to_evaluation_row(
     Args:
         trace: Trace dictionary from Fireworks proxy API
         include_tool_calls: Whether to include tool calling information
-        span_name: If provided, extract messages from generations within this named span
+        span_name: Not supported by this converter. Each row returned by the
+            Fireworks tracing endpoint is a single LLM call, so there is no
+            nested-span structure to walk. Pass a custom ``TraceDictConverter``
+            via ``get_evaluation_rows(..., converter=...)`` if you need
+            span-specific extraction logic.
 
     Returns:
         EvaluationRow or None if conversion fails
+
+    Raises:
+        NotImplementedError: If ``span_name`` is provided.
     """
+    if span_name:
+        raise NotImplementedError(
+            "span_name is not supported by the default Fireworks tracing converter. "
+            "Each trace row is already a single LLM call; provide a custom "
+            "TraceDictConverter to get_evaluation_rows() for span-aware logic."
+        )
     try:
         # Extract messages from trace input and output
         messages = extract_messages_from_trace_dict(trace, include_tool_calls, span_name)
@@ -125,97 +138,44 @@ def convert_trace_dict_to_evaluation_row(
 def extract_messages_from_trace_dict(
     trace: Dict[str, Any], include_tool_calls: bool = True, span_name: Optional[str] = None
 ) -> List[Message]:
-    """Extract messages from trace dictionary.
+    """Extract messages from a Fireworks trace row.
+
+    The Fireworks tracing endpoint returns one row per LLM call with the
+    request on ``Input`` and the response on ``Output``. There is no nested
+    span/generation structure at the row level.
 
     Args:
         trace: Trace dictionary from proxy API
         include_tool_calls: Whether to include tool calling information
-        span_name: If provided, extract messages from generations within this named span
+        span_name: Not supported. Pass a custom ``TraceDictConverter`` to
+            ``FireworksTracingAdapter.get_evaluation_rows`` if you need
+            span-specific extraction.
 
     Returns:
         List of Message objects
-    """
-    messages = []
-
-    if span_name:  # Look for a generation tied to a span name
-        try:
-            # Find the final generation in the named span
-            gen = get_final_generation_in_span_dict(trace, span_name)
-            if not gen:
-                return messages
-
-            # Extract messages from generation input and output
-            if gen.get("input"):
-                messages.extend(extract_messages_from_data(gen["input"], include_tool_calls))
-            if gen.get("output"):
-                messages.extend(extract_messages_from_data(gen["output"], include_tool_calls))
-
-            return messages
 
-        except Exception as e:
-            logger.error("Failed to extract messages from span '%s' in trace %s: %s", span_name, trace.get("id"), e)
-            return messages
-
-    else:
-        try:
-            # `Input` carries the request messages; `Output` carries the
-            # assistant message returned for this call. `extract_messages_from_data`
-            # accepts both `{"messages": [...]}` and single message dicts.
-            if trace.get("Input"):
-                messages.extend(extract_messages_from_data(trace["Input"], include_tool_calls))
-            if trace.get("Output"):
-                messages.extend(extract_messages_from_data(trace["Output"], include_tool_calls))
-        except (AttributeError, ValueError, KeyError) as e:
-            logger.warning("Error processing trace %s: %s", trace.get("InsertionId"), e)
-
-    return messages
-
-
-def get_final_generation_in_span_dict(trace: Dict[str, Any], span_name: str) -> Optional[Dict[str, Any]]:
-    """Get the final generation within a named span from trace dictionary.
-
-    Args:
-        trace: Trace dictionary
-        span_name: Name of the span to search for
-
-    Returns:
-        The final generation dictionary, or None if not found
+    Raises:
+        NotImplementedError: If ``span_name`` is provided.
     """
-    # Get all observations from the trace
-    all_observations = trace.get("observations", [])
-
-    # Find a span with the given name that has generation children
-    parent_span = None
-    for obs in all_observations:
-        if obs.get("name") == span_name and obs.get("type") == "SPAN":
-            # Check if this span has generation children
-            has_generations = any(
-                child.get("type") == "GENERATION" and child.get("parent_observation_id") == obs.get("id")
-                for child in all_observations
-            )
-            if has_generations:
-                parent_span = obs
-                break
-
-    if not parent_span:
-        logger.warning("No span named '%s' found in trace %s", span_name, trace.get("id"))
-        return None
-
-    # Find all generations within this span
-    generations = []
-    for obs in all_observations:
-        if obs.get("type") == "GENERATION" and obs.get("parent_observation_id") == parent_span.get("id"):
-            generations.append(obs)
-
-    if not generations:
-        logger.warning("No generations found in span '%s' in trace %s", span_name, trace.get("id"))
-        return None
+    if span_name:
+        raise NotImplementedError(
+            "span_name is not supported by extract_messages_from_trace_dict for "
+            "Fireworks traces; each row is already a single LLM call."
+        )
 
-    # Sort generations by start time for chronological order
-    generations.sort(key=lambda x: x.get("start_time", ""))
+    messages: List[Message] = []
+    try:
+        # `Input` carries the request messages; `Output` carries the
+        # assistant message returned for this call. `extract_messages_from_data`
+        # accepts both `{"messages": [...]}` and single message dicts.
+        if trace.get("Input"):
+            messages.extend(extract_messages_from_data(trace["Input"], include_tool_calls))
+        if trace.get("Output"):
+            messages.extend(extract_messages_from_data(trace["Output"], include_tool_calls))
+    except (AttributeError, ValueError, KeyError) as e:
+        logger.warning("Error processing trace %s: %s", trace.get("InsertionId"), e)
 
-    # Return the final generation (contains full message history)
-    return generations[-1]
+    return messages
 
 
 class FireworksTracingAdapter(BaseAdapter):
@@ -427,19 +387,29 @@ def get_evaluation_rows(
             include_tool_calls: Whether to include tool calling traces
             sleep_between_gets: Sleep time between polling attempts (default: 2.5s)
             max_retries: Max retry attempts used by proxy (default: 3)
+            span_name: Only supported when a custom ``converter`` is supplied.
+                The default Fireworks converter does not walk nested spans
+                (each trace row is already a single LLM call).
             converter: Optional custom converter implementing TraceDictConverter protocol.
                 If provided, this will be used instead of the default conversion logic.
 
         Returns:
             List[EvaluationRow]: Converted evaluation rows
 
         Raises:
-            ValueError: If tags list is empty
+            ValueError: If tags list is empty or no ``rollout_id`` tag is present.
+            NotImplementedError: If ``span_name`` is provided without a custom ``converter``.
         """
         # Validate that tags are provided
         if not tags or len(tags) == 0:
             raise ValueError("At least one tag is required to fetch traces")
 
+        if span_name and converter is None:
+            raise NotImplementedError(
+                "span_name is not supported by the default Fireworks tracing converter. "
+                "Pass a custom converter=TraceDictConverter(...) if you need span-aware logic."
+            )
+
         # Pull out rollout_id only, since that is the task-level id needed to fetch traces.
         rollout_id = next(
             (t.split(":", 1)[1] for t in tags if t.startswith("rollout_id:")),