cp dines

Author: dines-rl

docs/sdk/types.rst

@@ -81,16 +81,14 @@ These TypeDicts define parameters for storage object creation, listing, and down
 Scorer Parameters
 -----------------
 
-These TypeDicts define parameters for scorer creation, listing, updating, and validation.
+These TypeDicts define parameters for scorer creation, listing, and updating.
 
 .. autotypeddict:: runloop_api_client.sdk._types.SDKScorerCreateParams
 
 .. autotypeddict:: runloop_api_client.sdk._types.SDKScorerListParams
 
 .. autotypeddict:: runloop_api_client.sdk._types.SDKScorerUpdateParams
 
-.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerValidateParams
-
 Core Request Options
 --------------------
 

src/runloop_api_client/sdk/_types.py

@@ -37,7 +37,7 @@
 from .._types import Body, Query, Headers, Timeout, NotGiven
 from ..lib.polling import PollingConfig
 from ..types.devboxes import DiskSnapshotListParams, DiskSnapshotUpdateParams
-from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams, ScorerValidateParams
+from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams
 from ..types.devbox_create_params import DevboxBaseCreateParams
 from ..types.scenario_start_run_params import ScenarioStartRunBaseParams
 from ..types.benchmark_start_run_params import BenchmarkSelfStartRunParams
@@ -181,10 +181,6 @@ class SDKScorerUpdateParams(ScorerUpdateParams, LongRequestOptions):
     pass
 
 
-class SDKScorerValidateParams(ScorerValidateParams, LongRequestOptions):
-    pass
-
-
 class SDKAgentCreateParams(AgentCreateParams, LongRequestOptions):
     pass
 

src/runloop_api_client/sdk/async_scorer.py

@@ -7,10 +7,9 @@
 from ._types import (
     BaseRequestOptions,
     SDKScorerUpdateParams,
-    SDKScorerValidateParams,
 )
 from .._client import AsyncRunloop
-from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse
+from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse
 
 
 class AsyncScorer:
@@ -66,12 +65,3 @@ async def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateR
         :rtype: ScorerUpdateResponse
         """
         return await self._client.scenarios.scorers.update(self._id, **params)
-
-    async def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse:
-        """Run the scorer against the provided context and return the result.
-
-        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters
-        :return: Validation result with score
-        :rtype: ScorerValidateResponse
-        """
-        return await self._client.scenarios.scorers.validate(self._id, **params)

src/runloop_api_client/sdk/scorer.py

@@ -7,10 +7,9 @@
 from ._types import (
     BaseRequestOptions,
     SDKScorerUpdateParams,
-    SDKScorerValidateParams,
 )
 from .._client import Runloop
-from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse
+from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse
 
 
 class Scorer:
@@ -66,12 +65,3 @@ def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateRespons
         :rtype: ScorerUpdateResponse
         """
         return self._client.scenarios.scorers.update(self._id, **params)
-
-    def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse:
-        """Run the scorer against the provided context and return the result.
-
-        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters
-        :return: Validation result with score
-        :rtype: ScorerValidateResponse
-        """
-        return self._client.scenarios.scorers.validate(self._id, **params)

tests/sdk/test_async_scorer.py

@@ -49,21 +49,3 @@ async def test_update(self, mock_async_client: AsyncMock) -> None:
 
         assert result == update_response
         mock_async_client.scenarios.scorers.update.assert_awaited_once()
-
-    @pytest.mark.asyncio
-    async def test_validate(self, mock_async_client: AsyncMock) -> None:
-        """Test validate method."""
-        validate_response = SimpleNamespace(
-            name="test_scorer",
-            scoring_context={},
-            scoring_result=SimpleNamespace(score=0.95),
-        )
-        mock_async_client.scenarios.scorers.validate = AsyncMock(return_value=validate_response)
-
-        scorer = AsyncScorer(mock_async_client, "sco_123")
-        result = await scorer.validate(
-            scoring_context={"test": "context"},
-        )
-
-        assert result == validate_response
-        mock_async_client.scenarios.scorers.validate.assert_awaited_once()

tests/sdk/test_scorer.py

@@ -49,23 +49,3 @@ def test_update(self, mock_client: Mock) -> None:
             type="updated_scorer",
             bash_script="echo 'score=1.0'",
         )
-
-    def test_validate(self, mock_client: Mock) -> None:
-        """Test validate method."""
-        validate_response = SimpleNamespace(
-            name="test_scorer",
-            scoring_context={},
-            scoring_result=SimpleNamespace(score=0.95),
-        )
-        mock_client.scenarios.scorers.validate.return_value = validate_response
-
-        scorer = Scorer(mock_client, "sco_123")
-        result = scorer.validate(
-            scoring_context={"test": "context"},
-        )
-
-        assert result == validate_response
-        mock_client.scenarios.scorers.validate.assert_called_once_with(
-            "sco_123",
-            scoring_context={"test": "context"},
-        )

tests/smoketests/sdk/test_async_scorer.py

@@ -4,7 +4,6 @@
 
 import pytest
 
-from runloop_api_client import InternalServerError
 from runloop_api_client.sdk import AsyncRunloopSDK
 from tests.smoketests.utils import unique_name
 
@@ -71,24 +70,6 @@ async def test_scorer_update(self, async_sdk_client: AsyncRunloopSDK) -> None:
         assert info.type == updated_type
         assert info.bash_script == "echo 'score=1.0'"
 
-    @pytest.mark.timeout(ONE_MINUTE_TIMEOUT)
-    async def test_scorer_validate(self, async_sdk_client: AsyncRunloopSDK) -> None:
-        """Test validating a scorer."""
-        scorer_type = unique_name("sdk-async-scorer-validate")
-        scorer = await async_sdk_client.scorer.create(
-            type=scorer_type,
-            bash_script="echo 'score=1.0'",
-        )
-
-        try:
-            result = await scorer.validate(
-                scoring_context={},
-            )
-            assert result is not None
-        except InternalServerError:
-            # Backend may return 500 for validate endpoint - skip if this happens
-            pytest.skip("Backend returned 500 for scorer validate endpoint")
-
 
 class TestAsyncScorerListing:
     """Test async scorer listing and retrieval operations."""

tests/smoketests/sdk/test_scorer.py

@@ -4,7 +4,6 @@
 
 import pytest
 
-from runloop_api_client import InternalServerError
 from runloop_api_client.sdk import RunloopSDK
 from tests.smoketests.utils import unique_name
 
@@ -71,24 +70,6 @@ def test_scorer_update(self, sdk_client: RunloopSDK) -> None:
         assert info.type == updated_type
         assert info.bash_script == "echo 'score=1.0'"
 
-    @pytest.mark.timeout(ONE_MINUTE_TIMEOUT)
-    def test_scorer_validate(self, sdk_client: RunloopSDK) -> None:
-        """Test validating a scorer."""
-        scorer_type = unique_name("sdk-scorer-validate")
-        scorer = sdk_client.scorer.create(
-            type=scorer_type,
-            bash_script="echo 'score=1.0'",
-        )
-
-        try:
-            result = scorer.validate(
-                scoring_context={},
-            )
-            assert result is not None
-        except InternalServerError:
-            # Backend may return 500 for validate endpoint - skip if this happens
-            pytest.skip("Backend returned 500 for scorer validate endpoint")
-
 
 class TestScorerListing:
     """Test scorer listing and retrieval operations."""