feat(sdk): added scorer classes to sdk (#698)

* added scorer class (kept create and list as static methods for now since we don't know how we're creating scorers yet)

* refactored static methods to ScorerOps class

* fix example docstrings to use correct scorer create params

* scorer tests

* fixed scorer unit test parameters for update and validate

* update scorer and scorer ops docstrings to be more helpful while not exposing system internals

* update docs with scorer classes, methods and types

* remove verbose request options in unit test parameters

* rename client to ops in client test

* rename client test file to ops

* added list_empty, list_single and list_multiple unit tests to all ops class tests

* fix assert_called to assert_awaited

* remove duplicate tests

Author: sid-rl

docs/sdk/async/index.rst

@@ -26,4 +26,5 @@ Asynchronous resource classes for working with devboxes, blueprints, snapshots,
    blueprint
    snapshot
    storage_object
+   scorer
 

docs/sdk/async/scorer.rst

@@ -0,0 +1,9 @@
+Scorer
+======
+
+The ``AsyncScorer`` class provides asynchronous methods for managing custom scenario scorers.
+
+.. automodule:: runloop_api_client.sdk.async_scorer
+   :members:
+
+

docs/sdk/sync/index.rst

@@ -26,4 +26,5 @@ Synchronous resource classes for working with devboxes, blueprints, snapshots, a
    blueprint
    snapshot
    storage_object
+   scorer
 

docs/sdk/sync/scorer.rst

@@ -0,0 +1,9 @@
+Scorer
+======
+
+The ``Scorer`` class provides synchronous methods for managing custom scenario scorers.
+
+.. automodule:: runloop_api_client.sdk.scorer
+   :members:
+
+

docs/sdk/types.rst

@@ -78,6 +78,19 @@ These TypeDicts define parameters for storage object creation, listing, and down
 
 .. autotypeddict:: runloop_api_client.sdk._types.SDKObjectDownloadParams
 
+Scorer Parameters
+-----------------
+
+These TypeDicts define parameters for scorer creation, listing, updating, and validation.
+
+.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerCreateParams
+
+.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerListParams
+
+.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerUpdateParams
+
+.. autotypeddict:: runloop_api_client.sdk._types.SDKScorerValidateParams
+
 Core Request Options
 --------------------
 

src/runloop_api_client/sdk/__init__.py

@@ -5,19 +5,22 @@
 
 from __future__ import annotations
 
-from .sync import DevboxOps, RunloopSDK, SnapshotOps, BlueprintOps, StorageObjectOps
+from .sync import DevboxOps, ScorerOps, RunloopSDK, SnapshotOps, BlueprintOps, StorageObjectOps
 from .async_ import (
     AsyncDevboxOps,
+    AsyncScorerOps,
     AsyncRunloopSDK,
     AsyncSnapshotOps,
     AsyncBlueprintOps,
     AsyncStorageObjectOps,
 )
 from .devbox import Devbox, NamedShell
+from .scorer import Scorer
 from .snapshot import Snapshot
 from .blueprint import Blueprint
 from .execution import Execution
 from .async_devbox import AsyncDevbox, AsyncNamedShell
+from .async_scorer import AsyncScorer
 from .async_snapshot import AsyncSnapshot
 from .storage_object import StorageObject
 from .async_blueprint import AsyncBlueprint
@@ -35,6 +38,8 @@
     "AsyncDevboxOps",
     "BlueprintOps",
     "AsyncBlueprintOps",
+    "ScorerOps",
+    "AsyncScorerOps",
     "SnapshotOps",
     "AsyncSnapshotOps",
     "StorageObjectOps",
@@ -48,6 +53,8 @@
     "AsyncExecutionResult",
     "Blueprint",
     "AsyncBlueprint",
+    "Scorer",
+    "AsyncScorer",
     "Snapshot",
     "AsyncSnapshot",
     "StorageObject",

src/runloop_api_client/sdk/_types.py

@@ -4,6 +4,7 @@
 from .._types import Body, Query, Headers, Timeout, NotGiven
 from ..lib.polling import PollingConfig
 from ..types.devboxes import DiskSnapshotListParams, DiskSnapshotUpdateParams
+from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams, ScorerValidateParams
 from ..types.devbox_list_params import DevboxListParams
 from ..types.object_list_params import ObjectListParams
 from ..types.devbox_create_params import DevboxCreateParams, DevboxBaseCreateParams
@@ -140,3 +141,19 @@ class SDKObjectCreateParams(ObjectCreateParams, LongRequestOptions):
 
 class SDKObjectDownloadParams(ObjectDownloadParams, BaseRequestOptions):
     pass
+
+
+class SDKScorerCreateParams(ScorerCreateParams, LongRequestOptions):
+    pass
+
+
+class SDKScorerListParams(ScorerListParams, BaseRequestOptions):
+    pass
+
+
+class SDKScorerUpdateParams(ScorerUpdateParams, LongRequestOptions):
+    pass
+
+
+class SDKScorerValidateParams(ScorerValidateParams, LongRequestOptions):
+    pass

src/runloop_api_client/sdk/async_.py

@@ -16,8 +16,10 @@
     LongRequestOptions,
     SDKDevboxListParams,
     SDKObjectListParams,
+    SDKScorerListParams,
     SDKDevboxCreateParams,
     SDKObjectCreateParams,
+    SDKScorerCreateParams,
     SDKBlueprintListParams,
     SDKBlueprintCreateParams,
     SDKDiskSnapshotListParams,
@@ -27,6 +29,7 @@
 from .._client import DEFAULT_MAX_RETRIES, AsyncRunloop
 from ._helpers import detect_content_type
 from .async_devbox import AsyncDevbox
+from .async_scorer import AsyncScorer
 from .async_snapshot import AsyncSnapshot
 from .async_blueprint import AsyncBlueprint
 from .async_storage_object import AsyncStorageObject
@@ -492,6 +495,54 @@ async def upload_from_bytes(
         return obj
 
 
+class AsyncScorerOps:
+    """Create and manage custom scorers (async). Access via ``runloop.scorer``.
+
+    Example:
+        >>> runloop = AsyncRunloopSDK()
+        >>> scorer = await runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'")
+        >>> all_scorers = await runloop.scorer.list()
+    """
+
+    def __init__(self, client: AsyncRunloop) -> None:
+        """Initialize AsyncScorerOps.
+
+        :param client: AsyncRunloop client instance
+        :type client: AsyncRunloop
+        """
+        self._client = client
+
+    async def create(self, **params: Unpack[SDKScorerCreateParams]) -> AsyncScorer:
+        """Create a new scorer with the given type and bash script.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerCreateParams` for available parameters
+        :return: The newly created scorer
+        :rtype: AsyncScorer
+        """
+        response = await self._client.scenarios.scorers.create(**params)
+        return AsyncScorer(self._client, response.id)
+
+    def from_id(self, scorer_id: str) -> AsyncScorer:
+        """Get an AsyncScorer instance for an existing scorer ID.
+
+        :param scorer_id: ID of the scorer
+        :type scorer_id: str
+        :return: AsyncScorer instance for the given ID
+        :rtype: AsyncScorer
+        """
+        return AsyncScorer(self._client, scorer_id)
+
+    async def list(self, **params: Unpack[SDKScorerListParams]) -> list[AsyncScorer]:
+        """List all scorers, optionally filtered by parameters.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerListParams` for available parameters
+        :return: List of scorers
+        :rtype: list[AsyncScorer]
+        """
+        page = await self._client.scenarios.scorers.list(**params)
+        return [AsyncScorer(self._client, item.id) async for item in page]
+
+
 class AsyncRunloopSDK:
     """High-level asynchronous entry point for the Runloop SDK.
 
@@ -505,6 +556,8 @@ class AsyncRunloopSDK:
     :vartype devbox: AsyncDevboxOps
     :ivar blueprint: High-level async interface for blueprint management
     :vartype blueprint: AsyncBlueprintOps
+    :ivar scorer: High-level async interface for scorer management
+    :vartype scorer: AsyncScorerOps
     :ivar snapshot: High-level async interface for snapshot management
     :vartype snapshot: AsyncSnapshotOps
     :ivar storage_object: High-level async interface for storage object management
@@ -521,6 +574,7 @@ class AsyncRunloopSDK:
     api: AsyncRunloop
     devbox: AsyncDevboxOps
     blueprint: AsyncBlueprintOps
+    scorer: AsyncScorerOps
     snapshot: AsyncSnapshotOps
     storage_object: AsyncStorageObjectOps
 
@@ -564,6 +618,7 @@ def __init__(
 
         self.devbox = AsyncDevboxOps(self.api)
         self.blueprint = AsyncBlueprintOps(self.api)
+        self.scorer = AsyncScorerOps(self.api)
         self.snapshot = AsyncSnapshotOps(self.api)
         self.storage_object = AsyncStorageObjectOps(self.api)
 

src/runloop_api_client/sdk/async_scorer.py

@@ -0,0 +1,77 @@
+"""Scorer resource class for asynchronous operations."""
+
+from __future__ import annotations
+
+from typing_extensions import Unpack, override
+
+from ._types import (
+    BaseRequestOptions,
+    SDKScorerUpdateParams,
+    SDKScorerValidateParams,
+)
+from .._client import AsyncRunloop
+from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse
+
+
+class AsyncScorer:
+    """A custom scorer for evaluating scenario outputs (async).
+
+    Scorers define bash scripts that produce a score (0.0-1.0) for scenario runs.
+    Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``.
+
+    Example:
+        >>> runloop = AsyncRunloopSDK()
+        >>> scorer = await runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'")
+        >>> await scorer.validate(scoring_context={"output": "test"})
+    """
+
+    def __init__(self, client: AsyncRunloop, scorer_id: str) -> None:
+        """Create an AsyncScorer instance.
+
+        :param client: AsyncRunloop client instance
+        :type client: AsyncRunloop
+        :param scorer_id: ID of the scorer
+        :type scorer_id: str
+        """
+        self._client = client
+        self._id = scorer_id
+
+    @override
+    def __repr__(self) -> str:
+        return f"<AsyncScorer id={self._id!r}>"
+
+    @property
+    def id(self) -> str:
+        """The scorer's unique identifier.
+
+        :return: Scorer ID
+        :rtype: str
+        """
+        return self._id
+
+    async def get_info(self, **options: Unpack[BaseRequestOptions]) -> ScorerRetrieveResponse:
+        """Fetch current scorer details from the API.
+
+        :param options: See :typeddict:`~runloop_api_client.sdk._types.BaseRequestOptions` for available options
+        :return: Current scorer details
+        :rtype: ScorerRetrieveResponse
+        """
+        return await self._client.scenarios.scorers.retrieve(self._id, **options)
+
+    async def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateResponse:
+        """Update the scorer's type or bash script.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerUpdateParams` for available parameters
+        :return: Updated scorer details
+        :rtype: ScorerUpdateResponse
+        """
+        return await self._client.scenarios.scorers.update(self._id, **params)
+
+    async def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse:
+        """Run the scorer against the provided context and return the result.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters
+        :return: Validation result with score
+        :rtype: ScorerValidateResponse
+        """
+        return await self._client.scenarios.scorers.validate(self._id, **params)

src/runloop_api_client/sdk/scorer.py

@@ -0,0 +1,77 @@
+"""Scorer resource class for synchronous operations."""
+
+from __future__ import annotations
+
+from typing_extensions import Unpack, override
+
+from ._types import (
+    BaseRequestOptions,
+    SDKScorerUpdateParams,
+    SDKScorerValidateParams,
+)
+from .._client import Runloop
+from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse
+
+
+class Scorer:
+    """A custom scorer for evaluating scenario outputs.
+
+    Scorers define bash scripts that produce a score (0.0-1.0) for scenario runs.
+    Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``.
+
+    Example:
+        >>> runloop = RunloopSDK()
+        >>> scorer = runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'")
+        >>> scorer.validate(scoring_context={"output": "test"})
+    """
+
+    def __init__(self, client: Runloop, scorer_id: str) -> None:
+        """Create a Scorer instance.
+
+        :param client: Runloop client instance
+        :type client: Runloop
+        :param scorer_id: ID of the scorer
+        :type scorer_id: str
+        """
+        self._client = client
+        self._id = scorer_id
+
+    @override
+    def __repr__(self) -> str:
+        return f"<Scorer id={self._id!r}>"
+
+    @property
+    def id(self) -> str:
+        """The scorer's unique identifier.
+
+        :return: Scorer ID
+        :rtype: str
+        """
+        return self._id
+
+    def get_info(self, **options: Unpack[BaseRequestOptions]) -> ScorerRetrieveResponse:
+        """Fetch current scorer details from the API.
+
+        :param options: See :typeddict:`~runloop_api_client.sdk._types.BaseRequestOptions` for available options
+        :return: Current scorer details
+        :rtype: ScorerRetrieveResponse
+        """
+        return self._client.scenarios.scorers.retrieve(self._id, **options)
+
+    def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateResponse:
+        """Update the scorer's type or bash script.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerUpdateParams` for available parameters
+        :return: Updated scorer details
+        :rtype: ScorerUpdateResponse
+        """
+        return self._client.scenarios.scorers.update(self._id, **params)
+
+    def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse:
+        """Run the scorer against the provided context and return the result.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters
+        :return: Validation result with score
+        :rtype: ScorerValidateResponse
+        """
+        return self._client.scenarios.scorers.validate(self._id, **params)