removed custom scorers

Author: james-rl

src/runloop_api_client/resources/scenarios/scorers.py

@@ -16,13 +16,11 @@
 )
 from ...pagination import SyncScenarioScorersCursorIDPage, AsyncScenarioScorersCursorIDPage
 from ..._base_client import AsyncPaginator, make_request_options
-from ...types.scenarios import scorer_list_params, scorer_create_params, scorer_update_params, scorer_validate_params
-from ...types.scenario_environment_param import ScenarioEnvironmentParam
+from ...types.scenarios import scorer_list_params, scorer_create_params, scorer_update_params
 from ...types.scenarios.scorer_list_response import ScorerListResponse
 from ...types.scenarios.scorer_create_response import ScorerCreateResponse
 from ...types.scenarios.scorer_update_response import ScorerUpdateResponse
 from ...types.scenarios.scorer_retrieve_response import ScorerRetrieveResponse
-from ...types.scenarios.scorer_validate_response import ScorerValidateResponse
 
 __all__ = ["ScorersResource", "AsyncScorersResource"]
 
@@ -232,59 +230,6 @@ def list(
             model=ScorerListResponse,
         )
 
-    def validate(
-        self,
-        id: str,
-        *,
-        scoring_context: object,
-        environment_parameters: ScenarioEnvironmentParam | Omit = omit,
-        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
-        # The extra values given here take precedence over values defined on the client or passed to this method.
-        extra_headers: Headers | None = None,
-        extra_query: Query | None = None,
-        extra_body: Body | None = None,
-        timeout: float | httpx.Timeout | None | NotGiven = not_given,
-        idempotency_key: str | None = None,
-    ) -> ScorerValidateResponse:
-        """
-        Validate a scenario scorer.
-
-        Args:
-          scoring_context: Json context that gets passed to the custom scorer
-
-          environment_parameters: The Environment in which the Scenario will run.
-
-          extra_headers: Send extra headers
-
-          extra_query: Add additional query parameters to the request
-
-          extra_body: Add additional JSON properties to the request
-
-          timeout: Override the client-level default timeout for this request, in seconds
-
-          idempotency_key: Specify a custom idempotency key for this request
-        """
-        if not id:
-            raise ValueError(f"Expected a non-empty value for `id` but received {id!r}")
-        return self._post(
-            f"/v1/scenarios/scorers/{id}/validate",
-            body=maybe_transform(
-                {
-                    "scoring_context": scoring_context,
-                    "environment_parameters": environment_parameters,
-                },
-                scorer_validate_params.ScorerValidateParams,
-            ),
-            options=make_request_options(
-                extra_headers=extra_headers,
-                extra_query=extra_query,
-                extra_body=extra_body,
-                timeout=timeout,
-                idempotency_key=idempotency_key,
-            ),
-            cast_to=ScorerValidateResponse,
-        )
-
 
 class AsyncScorersResource(AsyncAPIResource):
     @cached_property
@@ -491,59 +436,6 @@ def list(
             model=ScorerListResponse,
         )
 
-    async def validate(
-        self,
-        id: str,
-        *,
-        scoring_context: object,
-        environment_parameters: ScenarioEnvironmentParam | Omit = omit,
-        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
-        # The extra values given here take precedence over values defined on the client or passed to this method.
-        extra_headers: Headers | None = None,
-        extra_query: Query | None = None,
-        extra_body: Body | None = None,
-        timeout: float | httpx.Timeout | None | NotGiven = not_given,
-        idempotency_key: str | None = None,
-    ) -> ScorerValidateResponse:
-        """
-        Validate a scenario scorer.
-
-        Args:
-          scoring_context: Json context that gets passed to the custom scorer
-
-          environment_parameters: The Environment in which the Scenario will run.
-
-          extra_headers: Send extra headers
-
-          extra_query: Add additional query parameters to the request
-
-          extra_body: Add additional JSON properties to the request
-
-          timeout: Override the client-level default timeout for this request, in seconds
-
-          idempotency_key: Specify a custom idempotency key for this request
-        """
-        if not id:
-            raise ValueError(f"Expected a non-empty value for `id` but received {id!r}")
-        return await self._post(
-            f"/v1/scenarios/scorers/{id}/validate",
-            body=await async_maybe_transform(
-                {
-                    "scoring_context": scoring_context,
-                    "environment_parameters": environment_parameters,
-                },
-                scorer_validate_params.ScorerValidateParams,
-            ),
-            options=make_request_options(
-                extra_headers=extra_headers,
-                extra_query=extra_query,
-                extra_body=extra_body,
-                timeout=timeout,
-                idempotency_key=idempotency_key,
-            ),
-            cast_to=ScorerValidateResponse,
-        )
-
 
 class ScorersResourceWithRawResponse:
     def __init__(self, scorers: ScorersResource) -> None:
@@ -561,9 +453,6 @@ def __init__(self, scorers: ScorersResource) -> None:
         self.list = to_raw_response_wrapper(
             scorers.list,
         )
-        self.validate = to_raw_response_wrapper(
-            scorers.validate,
-        )
 
 
 class AsyncScorersResourceWithRawResponse:
@@ -582,9 +471,6 @@ def __init__(self, scorers: AsyncScorersResource) -> None:
         self.list = async_to_raw_response_wrapper(
             scorers.list,
         )
-        self.validate = async_to_raw_response_wrapper(
-            scorers.validate,
-        )
 
 
 class ScorersResourceWithStreamingResponse:
@@ -603,9 +489,6 @@ def __init__(self, scorers: ScorersResource) -> None:
         self.list = to_streamed_response_wrapper(
             scorers.list,
         )
-        self.validate = to_streamed_response_wrapper(
-            scorers.validate,
-        )
 
 
 class AsyncScorersResourceWithStreamingResponse:
@@ -624,6 +507,3 @@ def __init__(self, scorers: AsyncScorersResource) -> None:
         self.list = async_to_streamed_response_wrapper(
             scorers.list,
         )
-        self.validate = async_to_streamed_response_wrapper(
-            scorers.validate,
-        )

src/runloop_api_client/sdk/_types.py

@@ -37,7 +37,7 @@
 from .._types import Body, Query, Headers, Timeout, NotGiven
 from ..lib.polling import PollingConfig
 from ..types.devboxes import DiskSnapshotListParams, DiskSnapshotUpdateParams
-from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams, ScorerValidateParams
+from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams
 from ..types.devbox_create_params import DevboxBaseCreateParams
 from ..types.scenario_start_run_params import ScenarioStartRunBaseParams
 from ..types.benchmark_start_run_params import BenchmarkSelfStartRunParams
@@ -181,10 +181,6 @@ class SDKScorerUpdateParams(ScorerUpdateParams, LongRequestOptions):
     pass
 
 
-class SDKScorerValidateParams(ScorerValidateParams, LongRequestOptions):
-    pass
-
-
 class SDKAgentCreateParams(AgentCreateParams, LongRequestOptions):
     pass
 

src/runloop_api_client/sdk/async_scenario_builder.py

@@ -14,7 +14,6 @@
 from ..types.scoring_function_param import (
     Scorer,
     ScoringFunctionParam,
-    ScorerCustomScoringFunction,
     ScorerAstGrepScoringFunction,
     ScorerCommandScoringFunction,
     ScorerTestBasedScoringFunction,
@@ -304,35 +303,6 @@ def add_ast_grep_scorer(
             scorer["lang"] = lang
         return self._add_scorer(name, weight, scorer)
 
-    def add_custom_scorer(
-        self,
-        name: str,
-        *,
-        custom_scorer_type: str,
-        weight: float = 1.0,
-        scorer_params: Optional[object] = None,
-    ) -> Self:
-        """Add a custom scorer registered with Runloop.
-
-        :param name: Name of the scoring function
-        :type name: str
-        :param custom_scorer_type: Type identifier registered with Runloop
-        :type custom_scorer_type: str
-        :param weight: Weight for this scorer (normalized automatically)
-        :type weight: float
-        :param scorer_params: Additional JSON parameters for the scorer
-        :type scorer_params: Optional[object]
-        :return: Self for method chaining
-        :rtype: Self
-        """
-        scorer: ScorerCustomScoringFunction = {
-            "type": "custom_scorer",
-            "custom_scorer_type": custom_scorer_type,
-        }
-        if scorer_params:
-            scorer["scorer_params"] = scorer_params
-        return self._add_scorer(name, weight, scorer)
-
     def with_metadata(self, metadata: Dict[str, str]) -> Self:
         """Set metadata for the scenario.
 

src/runloop_api_client/sdk/async_scorer.py

@@ -7,22 +7,20 @@
 from ._types import (
     BaseRequestOptions,
     SDKScorerUpdateParams,
-    SDKScorerValidateParams,
 )
 from .._client import AsyncRunloop
-from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse
+from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse
 
 
 class AsyncScorer:
-    """A custom scorer for evaluating scenario outputs (async).
+    """A scorer for evaluating scenario outputs (async).
 
     Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs.
     Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``.
 
     Example:
         >>> runloop = AsyncRunloopSDK()
         >>> scorer = await runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'")
-        >>> await scorer.validate(scoring_context={"output": "test"})
     """
 
     def __init__(self, client: AsyncRunloop, scorer_id: str) -> None:
@@ -66,12 +64,3 @@ async def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateR
         :rtype: ScorerUpdateResponse
         """
         return await self._client.scenarios.scorers.update(self._id, **params)
-
-    async def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse:
-        """Run the scorer against the provided context and return the result.
-
-        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters
-        :return: Validation result with score
-        :rtype: ScorerValidateResponse
-        """
-        return await self._client.scenarios.scorers.validate(self._id, **params)

src/runloop_api_client/sdk/scenario_builder.py

@@ -14,7 +14,6 @@
 from ..types.scoring_function_param import (
     Scorer,
     ScoringFunctionParam,
-    ScorerCustomScoringFunction,
     ScorerAstGrepScoringFunction,
     ScorerCommandScoringFunction,
     ScorerTestBasedScoringFunction,
@@ -304,35 +303,6 @@ def add_ast_grep_scorer(
             scorer["lang"] = lang
         return self._add_scorer(name, weight, scorer)
 
-    def add_custom_scorer(
-        self,
-        name: str,
-        *,
-        custom_scorer_type: str,
-        weight: float = 1.0,
-        scorer_params: Optional[object] = None,
-    ) -> Self:
-        """Add a custom scorer registered with Runloop.
-
-        :param name: Name of the scoring function
-        :type name: str
-        :param custom_scorer_type: Type identifier registered with Runloop
-        :type custom_scorer_type: str
-        :param weight: Weight for this scorer (normalized automatically)
-        :type weight: float
-        :param scorer_params: Additional JSON parameters for the scorer
-        :type scorer_params: Optional[object]
-        :return: Self for method chaining
-        :rtype: Self
-        """
-        scorer: ScorerCustomScoringFunction = {
-            "type": "custom_scorer",
-            "custom_scorer_type": custom_scorer_type,
-        }
-        if scorer_params:
-            scorer["scorer_params"] = scorer_params
-        return self._add_scorer(name, weight, scorer)
-
     def with_metadata(self, metadata: Dict[str, str]) -> Self:
         """Set metadata for the scenario.
 

src/runloop_api_client/sdk/scorer.py

@@ -7,22 +7,20 @@
 from ._types import (
     BaseRequestOptions,
     SDKScorerUpdateParams,
-    SDKScorerValidateParams,
 )
 from .._client import Runloop
-from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse
+from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse
 
 
 class Scorer:
-    """A custom scorer for evaluating scenario outputs.
+    """A scorer for evaluating scenario outputs.
 
     Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs.
     Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``.
 
     Example:
         >>> runloop = RunloopSDK()
         >>> scorer = runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'")
-        >>> scorer.validate(scoring_context={"output": "test"})
     """
 
     def __init__(self, client: Runloop, scorer_id: str) -> None:
@@ -66,12 +64,3 @@ def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateRespons
         :rtype: ScorerUpdateResponse
         """
         return self._client.scenarios.scorers.update(self._id, **params)
-
-    def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse:
-        """Run the scorer against the provided context and return the result.
-
-        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters
-        :return: Validation result with score
-        :rtype: ScorerValidateResponse
-        """
-        return self._client.scenarios.scorers.validate(self._id, **params)

src/runloop_api_client/types/scenarios/__init__.py

@@ -9,6 +9,4 @@
 from .scorer_update_params import ScorerUpdateParams as ScorerUpdateParams
 from .scorer_create_response import ScorerCreateResponse as ScorerCreateResponse
 from .scorer_update_response import ScorerUpdateResponse as ScorerUpdateResponse
-from .scorer_validate_params import ScorerValidateParams as ScorerValidateParams
 from .scorer_retrieve_response import ScorerRetrieveResponse as ScorerRetrieveResponse
-from .scorer_validate_response import ScorerValidateResponse as ScorerValidateResponse