further consolidate scenario builder unit tests, make sure async coverage matches sync

Author: sid-rl

tests/sdk/test_async_scenario_builder.py

@@ -9,6 +9,7 @@
 from runloop_api_client.sdk.async_snapshot import AsyncSnapshot
 from runloop_api_client.sdk.async_blueprint import AsyncBlueprint
 from runloop_api_client.sdk.async_scenario_builder import AsyncScenarioBuilder
+from runloop_api_client.types.scoring_function_param import ScorerTestBasedScoringFunctionTestFile
 
 
 class TestAsyncScenarioBuilder:
@@ -37,17 +38,14 @@ def builder(self, mock_async_client: MagicMock) -> AsyncScenarioBuilder:
         """Create an AsyncScenarioBuilder instance with mock client."""
         return AsyncScenarioBuilder(mock_async_client, "test-scenario")
 
-    def test_init(self, mock_async_client: MagicMock) -> None:
-        """Test builder initialization."""
+    def test_instantiation(self, mock_async_client: MagicMock) -> None:
+        """Test builder initialization and repr."""
         builder = AsyncScenarioBuilder(mock_async_client, "my-scenario")
 
         assert builder._client is mock_async_client
         assert builder._name == "my-scenario"
         assert builder.name == "my-scenario"
-
-    def test_repr(self, builder: AsyncScenarioBuilder) -> None:
-        """Test builder __repr__."""
-        assert repr(builder) == "<AsyncScenarioBuilder name='test-scenario'>"
+        assert repr(builder) == "<AsyncScenarioBuilder name='my-scenario'>"
 
     def test_from_blueprint_and_snapshot(
         self, builder: AsyncScenarioBuilder, mock_blueprint: AsyncBlueprint, mock_snapshot: AsyncSnapshot
@@ -70,89 +68,122 @@ def test_from_blueprint_and_snapshot(
         assert builder._blueprint is mock_blueprint
         assert builder._snapshot is None
 
-    def test_with_working_directory_returns_self(self, builder: AsyncScenarioBuilder) -> None:
-        """Test with_working_directory returns self for chaining."""
-        result = builder.with_working_directory("/app")
-
-        assert result is builder
-        assert builder._working_directory == "/app"
-
-    def test_with_problem_statement_returns_self(self, builder: AsyncScenarioBuilder) -> None:
-        """Test with_problem_statement returns self for chaining."""
-        result = builder.with_problem_statement("Fix the bug")
-
-        assert result is builder
-        assert builder._problem_statement == "Fix the bug"
-
-    def test_add_test_scorer(self, builder: AsyncScenarioBuilder) -> None:
-        """Test add_test_scorer method."""
-        result = builder.add_test_scorer(
-            "my-tests",
-            test_command="pytest",
-            weight=2.0,
-        )
-
+    def test_scorers(self, builder: AsyncScenarioBuilder) -> None:
+        """Test all scorer types, optional params, and multiple scorers."""
+        # Test scorer with test files
+        test_files: list[ScorerTestBasedScoringFunctionTestFile] = [
+            {"file_path": "test_main.py", "file_contents": "def test_foo(): pass"}
+        ]
+        result = builder.add_test_scorer("test-scorer", test_command="pytest", weight=2.0, test_files=test_files)
         assert result is builder
-        assert len(builder._scorers) == 1
-        assert builder._scorers[0]["name"] == "my-tests"
+        assert builder._scorers[0]["name"] == "test-scorer"
+        assert builder._scorers[0]["weight"] == 2.0
         assert builder._scorers[0]["scorer"]["type"] == "test_based_scorer"
-
-    def test_add_command_scorer(self, builder: AsyncScenarioBuilder) -> None:
-        """Test add_command_scorer method."""
-        result = builder.add_command_scorer(
-            "cmd-scorer",
-            command="./check.sh",
+        assert builder._scorers[0]["scorer"].get("test_command") == "pytest"
+        assert builder._scorers[0]["scorer"].get("test_files") == test_files
+
+        # Command scorer
+        builder.add_command_scorer("cmd-scorer", command="./check.sh")
+        assert builder._scorers[1]["scorer"]["type"] == "command_scorer"
+        assert builder._scorers[1]["scorer"].get("command") == "./check.sh"
+
+        # Bash scorer
+        builder.add_bash_scorer("bash-scorer", bash_script="echo 'score=1.0'")
+        assert builder._scorers[2]["scorer"]["type"] == "bash_script_scorer"
+        assert builder._scorers[2]["scorer"].get("bash_script") == "echo 'score=1.0'"
+
+        # Python scorer with optional params
+        builder.add_python_scorer(
+            "python-scorer",
+            python_script="print('1.0')",
+            python_version_constraint=">=3.10",
+            requirements_contents="numpy",
         )
-
-        assert result is builder
-        assert builder._scorers[0]["scorer"]["type"] == "command_scorer"
-
-    def test_add_bash_scorer(self, builder: AsyncScenarioBuilder) -> None:
-        """Test add_bash_scorer method."""
-        result = builder.add_bash_scorer(
-            "bash-scorer",
-            bash_script="echo 'score=1.0'",
-        )
-
-        assert result is builder
-        assert builder._scorers[0]["scorer"]["type"] == "bash_script_scorer"
-
-    def test_build_params_missing_problem_statement(self, builder: AsyncScenarioBuilder) -> None:
-        """Test _build_params raises if problem statement is missing."""
+        assert builder._scorers[3]["scorer"]["type"] == "python_script_scorer"
+        assert builder._scorers[3]["scorer"].get("python_version_constraint") == ">=3.10"
+        assert builder._scorers[3]["scorer"].get("requirements_contents") == "numpy"
+
+        # AST grep scorer with optional lang
+        builder.add_ast_grep_scorer("ast-scorer", pattern="$A.foo()", search_directory="/src", lang="python")
+        assert builder._scorers[4]["scorer"]["type"] == "ast_grep_scorer"
+        assert builder._scorers[4]["scorer"].get("pattern") == "$A.foo()"
+        assert builder._scorers[4]["scorer"].get("lang") == "python"
+
+        # Custom scorer with optional params
+        builder.add_custom_scorer("custom-scorer", custom_scorer_type="my_scorer", scorer_params={"threshold": 0.5})
+        assert builder._scorers[5]["scorer"]["type"] == "custom_scorer"
+        assert builder._scorers[5]["scorer"].get("custom_scorer_type") == "my_scorer"
+        assert builder._scorers[5]["scorer"].get("scorer_params") == {"threshold": 0.5}
+
+        # Verify multiple scorers accumulated
+        assert len(builder._scorers) == 6
+
+    def test_add_scorer_rejects_invalid_weight(self, builder: AsyncScenarioBuilder) -> None:
+        """Test that adding a scorer with zero or negative weight raises ValueError."""
+        with pytest.raises(ValueError, match="Scorer weight must be positive"):
+            builder.add_bash_scorer("bad", bash_script="echo 1", weight=0.0)
+
+        with pytest.raises(ValueError, match="Scorer weight must be positive"):
+            builder.add_bash_scorer("bad", bash_script="echo 1", weight=-1.0)
+
+    def test_build_params_validation(self, builder: AsyncScenarioBuilder) -> None:
+        """Test _build_params raises for missing required fields."""
+        # Missing problem statement
         builder.add_test_scorer("test", test_command="pytest")
-
         with pytest.raises(ValueError, match="Problem statement is required"):
             builder._build_params()
 
-    def test_build_params_missing_scorer(self, builder: AsyncScenarioBuilder) -> None:
-        """Test _build_params raises if no scorers are added."""
-        builder.with_problem_statement("Fix the bug")
-
+        # Missing scorer (new builder)
+        builder2 = AsyncScenarioBuilder(builder._client, "test2")
+        builder2.with_problem_statement("Fix the bug")
         with pytest.raises(ValueError, match="At least one scorer is required"):
-            builder._build_params()
+            builder2._build_params()
 
-    def test_build_params_minimal(self, builder: AsyncScenarioBuilder) -> None:
-        """Test _build_params with minimal configuration."""
+    def test_build_params_with_all_options(self, builder: AsyncScenarioBuilder, mock_blueprint: AsyncBlueprint) -> None:
+        """Test _build_params with all optional fields set."""
         builder.with_problem_statement("Fix the bug")
+        builder.with_additional_context({"hint": "line 42"})
         builder.add_test_scorer("tests", test_command="pytest")
+        builder.from_blueprint(mock_blueprint)
+        builder.with_working_directory("/app")
+        builder.with_metadata({"team": "infra"})
+        builder.with_reference_output("diff content")
+        builder.with_required_env_vars(["API_KEY"])
+        builder.with_required_secrets(["db_pass"])
+        builder.with_validation_type("FORWARD")
 
         params = builder._build_params()
 
         assert params["name"] == "test-scenario"
         assert params["input_context"]["problem_statement"] == "Fix the bug"
-        assert len(params["scoring_contract"]["scoring_function_parameters"]) == 1
-
-    def test_build_params_with_environment(self, builder: AsyncScenarioBuilder, mock_blueprint: AsyncBlueprint) -> None:
-        """Test _build_params includes environment parameters."""
+        assert params["input_context"]["additional_context"] == {"hint": "line 42"}
+        assert params["environment_parameters"]["blueprint_id"] == "bp-123"
+        assert params["environment_parameters"]["working_directory"] == "/app"
+        assert params["metadata"] == {"team": "infra"}
+        assert params["reference_output"] == "diff content"
+        assert params["required_environment_variables"] == ["API_KEY"]
+        assert params["required_secret_names"] == ["db_pass"]
+        assert params["validation_type"] == "FORWARD"
+
+    def test_build_params_normalizes_weights(self, builder: AsyncScenarioBuilder) -> None:
+        """Test that _build_params normalizes scorer weights to sum to 1.0."""
         builder.with_problem_statement("Fix the bug")
-        builder.add_test_scorer("tests", test_command="pytest")
-        builder.from_blueprint(mock_blueprint)
-        builder.with_working_directory("/app")
+        builder.add_bash_scorer("scorer1", bash_script="echo 1", weight=1.0)
+        builder.add_bash_scorer("scorer2", bash_script="echo 2", weight=2.0)
+        builder.add_bash_scorer("scorer3", bash_script="echo 3", weight=3.0)
 
         params = builder._build_params()
+        scorers = params["scoring_contract"]["scoring_function_parameters"]
 
-        assert params["environment_parameters"]["blueprint_id"] == "bp-123"
-        assert params["environment_parameters"]["working_directory"] == "/app"
+        # Weights 1, 2, 3 should normalize to 1/6, 2/6, 3/6
+        assert len(scorers) == 3
+        assert abs(scorers[0]["weight"] - 1 / 6) < 0.0001
+        assert abs(scorers[1]["weight"] - 2 / 6) < 0.0001
+        assert abs(scorers[2]["weight"] - 3 / 6) < 0.0001
+
+        # Total should be 1.0
+        total = sum(s["weight"] for s in scorers)
+        assert abs(total - 1.0) < 0.0001
 
     @pytest.mark.asyncio
     async def test_push_calls_api_and_returns_scenario(