Merge branch 'main' into data-loader

Author: Dylan Huang

.vscode/settings.json

@@ -10,5 +10,8 @@
   "editor.formatOnSave": true,
   "[python]": {
     "editor.defaultFormatter": "charliermarsh.ruff"
+  },
+  "[typescript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
   }
 }

README.md

@@ -4,7 +4,18 @@
 
 **The open-source toolkit for building your internal model leaderboard.**
 
-When you have multiple AI models to choose from—different versions, providers, or configurations—how do you know which one is best for your use case?
+When you have multiple AI models to choose from—different versions, providers,
+or configurations—how do you know which one is best for your use case?
+
+## 🚀 Features
+
+- **Custom Evaluations**: Write evaluations tailored to your specific business needs
+- **Auto-Evaluation**: Stack-rank models using LLMs as judges with just model traces using out-of-the-box evaluators
+- **RL Environments via MCP**: Build reinforcement learning environments using the Model Control Protocol (MCP) to simulate user interactions and advanced evaluation scenarios
+- **Consistent Testing**: Test across various models and configurations with a unified framework
+- **Resilient Runtime**: Automatic retries for unstable LLM APIs and concurrent execution for long-running evaluations
+- **Rich Visualizations**: Built-in pivot tables and visualizations for result analysis
+- **Data-Driven Decisions**: Make informed model deployment decisions based on comprehensive evaluation results
 
 ## Quick Examples
 
@@ -69,15 +80,6 @@ def test_math_reasoning(row: EvaluationRow) -> EvaluationRow:
     return row
 ```
 
-## 🚀 Features
-
-- **Custom Evaluations**: Write evaluations tailored to your specific business needs
-- **Auto-Evaluation**: Stack-rank models using LLMs as judges with just model traces
-- **Model Context Protocol (MCP) Integration**: Build reinforcement learning environments and trigger user simulations for complex scenarios
-- **Consistent Testing**: Test across various models and configurations with a unified framework
-- **Resilient Runtime**: Automatic retries for unstable LLM APIs and concurrent execution for long-running evaluations
-- **Rich Visualizations**: Built-in pivot tables and visualizations for result analysis
-- **Data-Driven Decisions**: Make informed model deployment decisions based on comprehensive evaluation results
 
 ## 📚 Resources
 

eval_protocol/__init__.py

@@ -37,19 +37,19 @@
 from .resources import create_llm_resource
 from .reward_function import RewardFunction
 from .typed_interface import reward_function
-from .quickstart import aha_judge, split_multi_turn_rows
+from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assistant_to_ground_truth
 from .pytest import evaluation_test, SingleTurnRolloutProcessor
 from .pytest.parameterize import DefaultParameterIdGenerator
 
 from .adapters import OpenAIResponsesAdapter
 
 try:
-    from .adapters import LangfuseAdapter
+    from .adapters import LangfuseAdapter, create_langfuse_adapter
 except ImportError:
     LangfuseAdapter = None
 
 try:
-    from .adapters import BraintrustAdapter
+    from .adapters import BraintrustAdapter, create_braintrust_adapter
 except ImportError:
     BraintrustAdapter = None
 
@@ -64,12 +64,15 @@
 __all__ = [
     "DefaultParameterIdGenerator",
     "aha_judge",
-    "split_multi_turn_rows",
+    "multi_turn_assistant_to_ground_truth",
+    "assistant_to_ground_truth",
     "evaluation_test",
     "SingleTurnRolloutProcessor",
     "OpenAIResponsesAdapter",
     "LangfuseAdapter",
+    "create_langfuse_adapter",
     "BraintrustAdapter",
+    "create_braintrust_adapter",
     "LangSmithAdapter",
     # Core interfaces
     "Message",

eval_protocol/adapters/base.py

@@ -19,3 +19,7 @@ def get_evaluation_rows(self, *args, **kwargs) -> List[EvaluationRow]:
     def upload_scores(self, rows: List[EvaluationRow], model_name: str, mean_score: float) -> None:
         """Upload evaluation scores back to the data source for tracking and analysis."""
         pass
+
+    def upload_score(self, row: EvaluationRow, model_name: str) -> None:
+        """Upload evaluation score for a single row back to the data source."""
+        pass

eval_protocol/adapters/braintrust.py

@@ -264,6 +264,40 @@ def upload_scores(self, rows: List[EvaluationRow], model_name: str, mean_score:
         except Exception as e:
             logger.warning("Failed to push scores to Braintrust: %s", e)
 
+    def upload_score(self, row: EvaluationRow, model_name: str) -> None:
+        """Upload evaluation score for a single row back to Braintrust.
+
+        Args:
+            row: Single EvaluationRow with evaluation_result and session_data containing trace ID
+            model_name: Name of the model (used as the score name in Braintrust)
+        """
+        try:
+            if (
+                row.evaluation_result
+                and row.evaluation_result.is_score_valid
+                and row.input_metadata
+                and row.input_metadata.session_data
+                and "braintrust_trace_id" in row.input_metadata.session_data
+            ):
+                headers = {
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Content-Type": "application/json",
+                }
+
+                trace_id = row.input_metadata.session_data["braintrust_trace_id"]
+                if trace_id:
+                    feedback_items = [{"id": trace_id, "scores": {model_name: row.evaluation_result.score}}]
+
+                    response = requests.post(
+                        f"{self.api_url}/v1/feedback",
+                        headers=headers,
+                        json={"feedback": feedback_items},
+                        timeout=30,
+                    )
+                    response.raise_for_status()
+        except Exception as e:
+            logger.warning("Failed to upload single score to Braintrust: %s", e)
+
 
 def create_braintrust_adapter(
     api_key: Optional[str] = None,