handle base_experiment_name comparison

Author: barrettpyke

py/src/braintrust/framework.py

@@ -1351,6 +1351,14 @@ def _validate_classification_result(value: Any, classifier_name: str) -> Classif
     return classification
 
 
+def _get_persisted_base_experiment_id(experiment: Experiment) -> str | None:
+    try:
+        base_experiment_id = experiment.data.get("base_exp_id")
+    except Exception:
+        return None
+    return base_experiment_id if isinstance(base_experiment_id, str) and base_experiment_id else None
+
+
 async def run_evaluator(
     experiment: Experiment | None,
     evaluator: Evaluator[Input, Output, Expected],
@@ -1367,9 +1375,12 @@ async def run_evaluator(
     )
 
     if experiment:
+        comparison_experiment_id = evaluator.base_experiment_id
+        if comparison_experiment_id is None:
+            comparison_experiment_id = _get_persisted_base_experiment_id(experiment)
         summary = experiment.summarize(
             summarize_scores=evaluator.summarize_scores,
-            comparison_experiment_id=evaluator.base_experiment_id,
+            comparison_experiment_id=comparison_experiment_id,
         )
     else:
         summary = build_local_summary(evaluator, results)

py/src/braintrust/test_framework.py

@@ -107,6 +107,36 @@ def exact_match(input_value, output, expected):
     )
 
 
+@pytest.mark.asyncio
+async def test_run_evaluator_forwards_persisted_base_experiment_id_to_summary(with_memory_logger, with_simulate_login):
+    def exact_match(input_value, output, expected):
+        return 1.0 if output == expected else 0.0
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-evaluator",
+        data=[EvalCase(input=1, expected=1)],
+        task=lambda input_value: input_value,
+        scores=[exact_match],
+        experiment_name=None,
+        metadata=None,
+        base_experiment_name="base-exp",
+    )
+
+    exp = init_test_exp("test-evaluator", "test-project")
+    exp.data["base_exp_id"] = "base-exp-id"
+    expected_summary = MagicMock()
+    exp.summarize = MagicMock(return_value=expected_summary)
+
+    result = await run_evaluator(experiment=exp, evaluator=evaluator, position=None, filters=[])
+
+    assert result.summary is expected_summary
+    exp.summarize.assert_called_once_with(
+        summarize_scores=True,
+        comparison_experiment_id="base-exp-id",
+    )
+
+
 def test_experiment_summarize_resolves_explicit_comparison_name(with_memory_logger, with_simulate_login):
     exp = init_test_exp("test-evaluator", "test-project")
     mock_conn = MagicMock()