added preview() method to scenario builder

Author: sid-rl

src/runloop_api_client/sdk/_types.py

@@ -1,4 +1,4 @@
-from typing import Dict, Union, Callable, Optional
+from typing import Union, Callable, Optional
 from typing_extensions import TypedDict
 
 from .._types import Body, Query, Headers, Timeout, NotGiven
@@ -7,7 +7,6 @@
 from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams, ScorerValidateParams
 from ..types.input_context import InputContext
 from ..types.scenario_view import ScenarioView
-from ..types.scoring_contract import ScoringContract
 from ..types.agent_list_params import AgentListParams
 from ..types.devbox_list_params import DevboxListParams
 from ..types.object_list_params import ObjectListParams
@@ -202,14 +201,5 @@ class ScenarioPreview(ScenarioView):
     id: Optional[str] = None  # type: ignore[assignment]
     """The ID of the Scenario."""
 
-    input_context: Optional[InputContextPreview] = None  # type: ignore[assignment]
+    input_context: InputContextPreview  # type: ignore[assignment]
     """The input context for the Scenario."""
-
-    metadata: Optional[Dict[str, str]] = None  # type: ignore[assignment]
-    """User defined metadata to attach to the scenario for organization."""
-
-    name: Optional[str] = None  # type: ignore[assignment]
-    """The name of the Scenario."""
-
-    scoring_contract: Optional[ScoringContract] = None  # type: ignore[assignment]
-    """The scoring contract for the Scenario."""

src/runloop_api_client/sdk/async_scenario_builder.py

@@ -65,7 +65,7 @@ def __init__(self, name: str, client: AsyncRunloop) -> None:
         self._scorers: List[ScoringFunctionParam] = []
 
         # Metadata and other options
-        self._metadata: Optional[Dict[str, str]] = None
+        self._metadata: Dict[str, str] = {}
         self._reference_output: Optional[str] = None
         self._required_env_vars: Optional[List[str]] = None
         self._required_secrets: Optional[List[str]] = None
@@ -392,16 +392,15 @@ def _build_normalized_scorers(self) -> List[ScoringFunctionParam]:
         total_weight = sum(s["weight"] for s in self._scorers)
         return [{**s, "weight": s["weight"] / total_weight} for s in self._scorers]
 
-    def _build_environment_params(self) -> ScenarioEnvironmentParam:
+    def _build_environment_params(self) -> Optional[ScenarioEnvironmentParam]:
         """Build environment parameters."""
-        env_params: ScenarioEnvironmentParam = {}
-        if self._blueprint:
-            env_params["blueprint_id"] = self._blueprint.id
-        if self._snapshot:
-            env_params["snapshot_id"] = self._snapshot.id
-        if self._working_directory:
-            env_params["working_directory"] = self._working_directory
-        return env_params
+        if not self._blueprint and not self._snapshot and not self._working_directory:
+            return None
+        return {
+            "blueprint_id": self._blueprint.id if self._blueprint else None,
+            "snapshot_id": self._snapshot.id if self._snapshot else None,
+            "working_directory": self._working_directory if self._working_directory else None,
+        }
 
     def _build_params(self) -> ScenarioCreateParams:
         """Build the scenario creation parameters.

src/runloop_api_client/sdk/scenario_builder.py

@@ -65,7 +65,7 @@ def __init__(self, name: str, client: Runloop) -> None:
         self._scorers: List[ScoringFunctionParam] = []
 
         # Metadata and other options
-        self._metadata: Optional[Dict[str, str]] = None
+        self._metadata: Dict[str, str] = {}
         self._reference_output: Optional[str] = None
         self._required_env_vars: Optional[List[str]] = None
         self._required_secrets: Optional[List[str]] = None
@@ -392,16 +392,15 @@ def _build_normalized_scorers(self) -> List[ScoringFunctionParam]:
         total_weight = sum(s["weight"] for s in self._scorers)
         return [{**s, "weight": s["weight"] / total_weight} for s in self._scorers]
 
-    def _build_environment_params(self) -> ScenarioEnvironmentParam:
+    def _build_environment_params(self) -> Optional[ScenarioEnvironmentParam]:
         """Build environment parameters"""
-        env_params: ScenarioEnvironmentParam = {}
-        if self._blueprint:
-            env_params["blueprint_id"] = self._blueprint.id
-        if self._snapshot:
-            env_params["snapshot_id"] = self._snapshot.id
-        if self._working_directory:
-            env_params["working_directory"] = self._working_directory
-        return env_params
+        if not self._blueprint and not self._snapshot and not self._working_directory:
+            return None
+        return {
+            "blueprint_id": self._blueprint.id if self._blueprint else None,
+            "snapshot_id": self._snapshot.id if self._snapshot else None,
+            "working_directory": self._working_directory if self._working_directory else None,
+        }
 
     def _build_params(self) -> ScenarioCreateParams:
         """Build the scenario creation parameters.

tests/sdk/test_async_scenario_builder.py

@@ -6,7 +6,7 @@
 
 import pytest
 
-from runloop_api_client.sdk import AsyncSnapshot, AsyncBlueprint, AsyncScenarioBuilder
+from runloop_api_client.sdk import AsyncSnapshot, AsyncBlueprint, ScenarioPreview, AsyncScenarioBuilder
 from runloop_api_client.types.scoring_function_param import ScorerTestBasedScoringFunctionTestFile
 
 
@@ -230,3 +230,65 @@ def test_fluent_chaining(self, mock_builder: AsyncScenarioBuilder, mock_blueprin
         assert mock_builder._working_directory == "/app"
         assert mock_builder._problem_statement == "Fix the bug"
         assert len(mock_builder._scorers) == 1
+
+    def test_preview_with_no_config(self, mock_builder: AsyncScenarioBuilder) -> None:
+        """Test preview() works with no configuration (only name from constructor)."""
+        preview = mock_builder.preview()
+
+        assert isinstance(preview, ScenarioPreview)
+        assert preview.name == "test-scenario"
+        assert preview.input_context is not None
+        assert preview.input_context.problem_statement is None
+        assert preview.input_context.additional_context is None
+        assert preview.scoring_contract is not None
+        assert len(preview.scoring_contract.scoring_function_parameters) == 0
+        assert preview.environment is None
+        assert len(preview.metadata) == 0
+        assert preview.reference_output is None
+        assert preview.required_environment_variables is None
+        assert preview.required_secret_names is None
+        assert preview.validation_type is None
+
+    def test_preview_with_full_config(self, mock_builder: AsyncScenarioBuilder, mock_blueprint: AsyncBlueprint) -> None:
+        """Test preview() with all fields configured, including weight normalization."""
+        mock_builder.with_problem_statement("Fix the bug")
+        mock_builder.with_additional_context({"hint": "line 42"})
+        mock_builder.from_blueprint(mock_blueprint)
+        mock_builder.with_working_directory("/app")
+        mock_builder.with_metadata({"team": "infra"})
+        mock_builder.with_reference_output("diff content")
+        mock_builder.with_required_env_vars(["API_KEY"])
+        mock_builder.with_required_secrets(["db_pass"])
+        mock_builder.with_validation_type("FORWARD")
+        # Add multiple scorers with different weights to test normalization
+        mock_builder.add_bash_script_scorer("scorer1", bash_script="echo 1", weight=1.0)
+        mock_builder.add_bash_script_scorer("scorer2", bash_script="echo 2", weight=2.0)
+        mock_builder.add_bash_script_scorer("scorer3", bash_script="echo 3", weight=3.0)
+
+        preview = mock_builder.preview()
+
+        # Verify it returns ScenarioPreview
+        assert isinstance(preview, ScenarioPreview)
+
+        # Verify all fields are populated
+        assert preview.name == "test-scenario"
+        assert preview.input_context is not None
+        assert preview.input_context.problem_statement == "Fix the bug"
+        assert preview.input_context.additional_context == {"hint": "line 42"}
+        assert preview.environment is not None
+        assert preview.environment.blueprint_id == "bp-123"
+        assert preview.environment.working_directory == "/app"
+        assert preview.metadata == {"team": "infra"}
+        assert preview.reference_output == "diff content"
+        assert preview.required_environment_variables == ["API_KEY"]
+        assert preview.required_secret_names == ["db_pass"]
+        assert preview.validation_type == "FORWARD"
+
+        # Verify weights are normalized (1, 2, 3 -> 1/6, 2/6, 3/6)
+        assert preview.scoring_contract is not None
+        scorers = preview.scoring_contract.scoring_function_parameters
+        assert len(scorers) == 3
+        assert abs(scorers[0].weight - 1 / 6) < 0.0001
+        assert abs(scorers[1].weight - 2 / 6) < 0.0001
+        assert abs(scorers[2].weight - 3 / 6) < 0.0001
+        assert abs(sum(s.weight for s in scorers) - 1.0) < 0.0001

tests/sdk/test_scenario_builder.py

@@ -6,7 +6,7 @@
 
 import pytest
 
-from runloop_api_client.sdk import Snapshot, Blueprint, ScenarioBuilder
+from runloop_api_client.sdk import Snapshot, Blueprint, ScenarioBuilder, ScenarioPreview
 from runloop_api_client.types.scoring_function_param import ScorerTestBasedScoringFunctionTestFile
 
 
@@ -223,3 +223,65 @@ def test_fluent_chaining(self, mock_builder: ScenarioBuilder, mock_blueprint: Bl
         assert mock_builder._working_directory == "/app"
         assert mock_builder._problem_statement == "Fix the bug"
         assert len(mock_builder._scorers) == 1
+
+    def test_preview_with_no_config(self, mock_builder: ScenarioBuilder) -> None:
+        """Test preview() works with no configuration (only name from constructor)."""
+        preview = mock_builder.preview()
+
+        assert isinstance(preview, ScenarioPreview)
+        assert preview.name == "test-scenario"
+        assert preview.input_context is not None
+        assert preview.input_context.problem_statement is None
+        assert preview.input_context.additional_context is None
+        assert preview.scoring_contract is not None
+        assert len(preview.scoring_contract.scoring_function_parameters) == 0
+        assert preview.environment is None
+        assert len(preview.metadata) == 0
+        assert preview.reference_output is None
+        assert preview.required_environment_variables is None
+        assert preview.required_secret_names is None
+        assert preview.validation_type is None
+
+    def test_preview_with_full_config(self, mock_builder: ScenarioBuilder, mock_blueprint: Blueprint) -> None:
+        """Test preview() with all fields configured, including weight normalization."""
+        mock_builder.with_problem_statement("Fix the bug")
+        mock_builder.with_additional_context({"hint": "line 42"})
+        mock_builder.from_blueprint(mock_blueprint)
+        mock_builder.with_working_directory("/app")
+        mock_builder.with_metadata({"team": "infra"})
+        mock_builder.with_reference_output("diff content")
+        mock_builder.with_required_env_vars(["API_KEY"])
+        mock_builder.with_required_secrets(["db_pass"])
+        mock_builder.with_validation_type("FORWARD")
+        # Add multiple scorers with different weights to test normalization
+        mock_builder.add_bash_script_scorer("scorer1", bash_script="echo 1", weight=1.0)
+        mock_builder.add_bash_script_scorer("scorer2", bash_script="echo 2", weight=2.0)
+        mock_builder.add_bash_script_scorer("scorer3", bash_script="echo 3", weight=3.0)
+
+        preview = mock_builder.preview()
+
+        # Verify it returns ScenarioPreview
+        assert isinstance(preview, ScenarioPreview)
+
+        # Verify all fields are populated
+        assert preview.name == "test-scenario"
+        assert preview.input_context is not None
+        assert preview.input_context.problem_statement == "Fix the bug"
+        assert preview.input_context.additional_context == {"hint": "line 42"}
+        assert preview.environment is not None
+        assert preview.environment.blueprint_id == "bp-123"
+        assert preview.environment.working_directory == "/app"
+        assert preview.metadata == {"team": "infra"}
+        assert preview.reference_output == "diff content"
+        assert preview.required_environment_variables == ["API_KEY"]
+        assert preview.required_secret_names == ["db_pass"]
+        assert preview.validation_type == "FORWARD"
+
+        # Verify weights are normalized (1, 2, 3 -> 1/6, 2/6, 3/6)
+        assert preview.scoring_contract is not None
+        scorers = preview.scoring_contract.scoring_function_parameters
+        assert len(scorers) == 3
+        assert abs(scorers[0].weight - 1 / 6) < 0.0001
+        assert abs(scorers[1].weight - 2 / 6) < 0.0001
+        assert abs(scorers[2].weight - 3 / 6) < 0.0001
+        assert abs(sum(s.weight for s in scorers) - 1.0) < 0.0001