feat(scenarios): added scenario class to sdk (#701)

* uv lock upgrade

* scenario wrapper

* modified ScenarioStartRunParams to follow our previous pattern for use in the SDK

* fixed redundancy and formatting in scenario and scenario run unit tests

* formatting fixes

* changed scenario and scenario smoketests to update existing scenarios and add metadata tag instead of creating new ones every time (workaround for lack of cleanup option)

* changed scenario run methods to `run` and `run_async`, updated parameter TypedDicts appropriately

* exposed all scenario update parameters appropriately using typeddict

* formatting fixes

* remove scenario builder smoketests

* add score_and_complete and download_logs methods to scenario runs, and update docstrings to not leak internals

* add unit tests for ScenarioOps and AsyncScenarioOps

* update tests for scenario.update and change test name for scenario.run and run_async

* add unit tests for download_logs and score_and_complete

* don't pass polling_config to get_info within await_env_ready

Author: sid-rl

src/runloop_api_client/sdk/__init__.py

@@ -5,30 +5,35 @@
 
 from __future__ import annotations
 
-from .sync import AgentOps, DevboxOps, ScorerOps, RunloopSDK, SnapshotOps, BlueprintOps, StorageObjectOps
+from .sync import AgentOps, DevboxOps, ScorerOps, RunloopSDK, ScenarioOps, SnapshotOps, BlueprintOps, StorageObjectOps
 from .agent import Agent
 from .async_ import (
     AsyncAgentOps,
     AsyncDevboxOps,
     AsyncScorerOps,
     AsyncRunloopSDK,
+    AsyncScenarioOps,
     AsyncSnapshotOps,
     AsyncBlueprintOps,
     AsyncStorageObjectOps,
 )
 from .devbox import Devbox, NamedShell
 from .scorer import Scorer
+from .scenario import Scenario
 from .snapshot import Snapshot
 from .blueprint import Blueprint
 from .execution import Execution
 from .async_agent import AsyncAgent
 from .async_devbox import AsyncDevbox, AsyncNamedShell
 from .async_scorer import AsyncScorer
+from .scenario_run import ScenarioRun
+from .async_scenario import AsyncScenario
 from .async_snapshot import AsyncSnapshot
 from .storage_object import StorageObject
 from .async_blueprint import AsyncBlueprint
 from .async_execution import AsyncExecution
 from .execution_result import ExecutionResult
+from .async_scenario_run import AsyncScenarioRun
 from .async_storage_object import AsyncStorageObject
 from .async_execution_result import AsyncExecutionResult
 
@@ -43,6 +48,8 @@
     "AsyncDevboxOps",
     "BlueprintOps",
     "AsyncBlueprintOps",
+    "ScenarioOps",
+    "AsyncScenarioOps",
     "ScorerOps",
     "AsyncScorerOps",
     "SnapshotOps",
@@ -60,6 +67,10 @@
     "AsyncExecutionResult",
     "Blueprint",
     "AsyncBlueprint",
+    "Scenario",
+    "AsyncScenario",
+    "ScenarioRun",
+    "AsyncScenarioRun",
     "Scorer",
     "AsyncScorer",
     "Snapshot",

src/runloop_api_client/sdk/_types.py

@@ -11,10 +11,13 @@
 from ..types.agent_create_params import AgentCreateParams
 from ..types.devbox_create_params import DevboxCreateParams, DevboxBaseCreateParams
 from ..types.object_create_params import ObjectCreateParams
+from ..types.scenario_list_params import ScenarioListParams
 from ..types.blueprint_list_params import BlueprintListParams
 from ..types.object_download_params import ObjectDownloadParams
+from ..types.scenario_update_params import ScenarioUpdateParams
 from ..types.blueprint_create_params import BlueprintCreateParams
 from ..types.devbox_upload_file_params import DevboxUploadFileParams
+from ..types.scenario_start_run_params import ScenarioStartRunBaseParams
 from ..types.devbox_create_tunnel_params import DevboxCreateTunnelParams
 from ..types.devbox_download_file_params import DevboxDownloadFileParams
 from ..types.devbox_execute_async_params import DevboxNiceExecuteAsyncParams
@@ -167,3 +170,19 @@ class SDKAgentCreateParams(AgentCreateParams, LongRequestOptions):
 
 class SDKAgentListParams(AgentListParams, BaseRequestOptions):
     pass
+
+
+class SDKScenarioListParams(ScenarioListParams, BaseRequestOptions):
+    pass
+
+
+class SDKScenarioUpdateParams(ScenarioUpdateParams, LongRequestOptions):
+    pass
+
+
+class SDKScenarioRunAsyncParams(ScenarioStartRunBaseParams, LongRequestOptions):
+    pass
+
+
+class SDKScenarioRunParams(ScenarioStartRunBaseParams, LongPollingRequestOptions):
+    pass

src/runloop_api_client/sdk/agent.py

@@ -19,10 +19,7 @@ class Agent:
     including retrieving agent information.
 
     Example:
-        >>> agent = runloop.agent.create_from_npm(
-        ...     name="my-agent",
-        ...     package_name="@runloop/example-agent"
-        ... )
+        >>> agent = runloop.agent.create_from_npm(name="my-agent", package_name="@runloop/example-agent")
         >>> info = agent.get_info()
         >>> print(info.name)
     """

src/runloop_api_client/sdk/async_.py

@@ -21,6 +21,7 @@
     SDKAgentCreateParams,
     SDKDevboxCreateParams,
     SDKObjectCreateParams,
+    SDKScenarioListParams,
     SDKScorerCreateParams,
     SDKBlueprintListParams,
     SDKBlueprintCreateParams,
@@ -33,6 +34,7 @@
 from .async_agent import AsyncAgent
 from .async_devbox import AsyncDevbox
 from .async_scorer import AsyncScorer
+from .async_scenario import AsyncScenario
 from .async_snapshot import AsyncSnapshot
 from .async_blueprint import AsyncBlueprint
 from .async_storage_object import AsyncStorageObject
@@ -543,7 +545,8 @@ async def list(self, **params: Unpack[SDKScorerListParams]) -> list[AsyncScorer]
         """
         page = await self._client.scenarios.scorers.list(**params)
         return [AsyncScorer(self._client, item.id) async for item in page]
-        
+
+
 class AsyncAgentOps:
     """High-level async manager for creating and managing agents.
 
@@ -553,15 +556,10 @@ class AsyncAgentOps:
     Example:
         >>> runloop = AsyncRunloopSDK()
         >>> # Create agent from NPM package
-        >>> agent = await runloop.agent.create_from_npm(
-        ...     name="my-agent",
-        ...     package_name="@runloop/example-agent"
-        ... )
+        >>> agent = await runloop.agent.create_from_npm(name="my-agent", package_name="@runloop/example-agent")
         >>> # Create agent from Git repository
         >>> agent = await runloop.agent.create_from_git(
-        ...     name="git-agent",
-        ...     repository="https://github.com/user/agent-repo",
-        ...     ref="main"
+        ...     name="git-agent", repository="https://github.com/user/agent-repo", ref="main"
         ... )
         >>> # List all agents
         >>> agents = await runloop.agent.list(limit=10)
@@ -615,7 +613,9 @@ async def create_from_npm(
         :raises ValueError: If 'source' is provided in params
         """
         if "source" in params:
-            raise ValueError("Cannot specify 'source' when using create_from_npm(); source is automatically set to npm configuration")
+            raise ValueError(
+                "Cannot specify 'source' when using create_from_npm(); source is automatically set to npm configuration"
+            )
 
         npm_config: dict = {"package_name": package_name}
         if npm_version is not None:
@@ -655,7 +655,9 @@ async def create_from_pip(
         :raises ValueError: If 'source' is provided in params
         """
         if "source" in params:
-            raise ValueError("Cannot specify 'source' when using create_from_pip(); source is automatically set to pip configuration")
+            raise ValueError(
+                "Cannot specify 'source' when using create_from_pip(); source is automatically set to pip configuration"
+            )
 
         pip_config: dict = {"package_name": package_name}
         if pip_version is not None:
@@ -692,7 +694,9 @@ async def create_from_git(
         :raises ValueError: If 'source' is provided in params
         """
         if "source" in params:
-            raise ValueError("Cannot specify 'source' when using create_from_git(); source is automatically set to git configuration")
+            raise ValueError(
+                "Cannot specify 'source' when using create_from_git(); source is automatically set to git configuration"
+            )
 
         git_config: dict = {"repository": repository}
         if ref is not None:
@@ -724,7 +728,9 @@ async def create_from_object(
         :raises ValueError: If 'source' is provided in params
         """
         if "source" in params:
-            raise ValueError("Cannot specify 'source' when using create_from_object(); source is automatically set to object configuration")
+            raise ValueError(
+                "Cannot specify 'source' when using create_from_object(); source is automatically set to object configuration"
+            )
 
         object_config: dict = {"object_id": object_id}
         if agent_setup is not None:
@@ -761,6 +767,45 @@ async def list(
         return [AsyncAgent(self._client, item.id, item) for item in page.agents]
 
 
+class AsyncScenarioOps:
+    """Manage scenarios (async). Access via ``runloop.scenario``.
+
+    Example:
+        >>> runloop = AsyncRunloopSDK()
+        >>> scenario = runloop.scenario.from_id("scn-xxx")
+        >>> run = await scenario.run()
+        >>> scenarios = await runloop.scenario.list()
+    """
+
+    def __init__(self, client: AsyncRunloop) -> None:
+        """Initialize AsyncScenarioOps.
+
+        :param client: AsyncRunloop client instance
+        :type client: AsyncRunloop
+        """
+        self._client = client
+
+    def from_id(self, scenario_id: str) -> AsyncScenario:
+        """Get an AsyncScenario instance for an existing scenario ID.
+
+        :param scenario_id: ID of the scenario
+        :type scenario_id: str
+        :return: AsyncScenario instance for the given ID
+        :rtype: AsyncScenario
+        """
+        return AsyncScenario(self._client, scenario_id)
+
+    async def list(self, **params: Unpack[SDKScenarioListParams]) -> list[AsyncScenario]:
+        """List all scenarios, optionally filtered by parameters.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScenarioListParams` for available parameters
+        :return: List of scenarios
+        :rtype: list[AsyncScenario]
+        """
+        page = await self._client.scenarios.list(**params)
+        return [AsyncScenario(self._client, item.id) async for item in page]
+
+
 class AsyncRunloopSDK:
     """High-level asynchronous entry point for the Runloop SDK.
 
@@ -776,6 +821,8 @@ class AsyncRunloopSDK:
     :vartype devbox: AsyncDevboxOps
     :ivar blueprint: High-level async interface for blueprint management
     :vartype blueprint: AsyncBlueprintOps
+    :ivar scenario: High-level async interface for scenario management
+    :vartype scenario: AsyncScenarioOps
     :ivar scorer: High-level async interface for scorer management
     :vartype scorer: AsyncScorerOps
     :ivar snapshot: High-level async interface for snapshot management
@@ -795,6 +842,7 @@ class AsyncRunloopSDK:
     agent: AsyncAgentOps
     devbox: AsyncDevboxOps
     blueprint: AsyncBlueprintOps
+    scenario: AsyncScenarioOps
     scorer: AsyncScorerOps
     snapshot: AsyncSnapshotOps
     storage_object: AsyncStorageObjectOps
@@ -840,6 +888,7 @@ def __init__(
         self.agent = AsyncAgentOps(self.api)
         self.devbox = AsyncDevboxOps(self.api)
         self.blueprint = AsyncBlueprintOps(self.api)
+        self.scenario = AsyncScenarioOps(self.api)
         self.scorer = AsyncScorerOps(self.api)
         self.snapshot = AsyncSnapshotOps(self.api)
         self.storage_object = AsyncStorageObjectOps(self.api)

src/runloop_api_client/sdk/async_agent.py

@@ -19,10 +19,7 @@ class AsyncAgent:
     including retrieving agent information.
 
     Example:
-        >>> agent = await runloop.agent.create_from_npm(
-        ...     name="my-agent",
-        ...     package_name="@runloop/example-agent"
-        ... )
+        >>> agent = await runloop.agent.create_from_npm(name="my-agent", package_name="@runloop/example-agent")
         >>> info = await agent.get_info()
         >>> print(info.name)
     """

src/runloop_api_client/sdk/async_scenario.py

@@ -0,0 +1,118 @@
+"""AsyncScenario resource class for asynchronous operations."""
+
+from __future__ import annotations
+
+from typing_extensions import Unpack, override
+
+from ..types import ScenarioView
+from ._types import BaseRequestOptions, SDKScenarioRunParams, SDKScenarioUpdateParams, SDKScenarioRunAsyncParams
+from .._client import AsyncRunloop
+from .async_scenario_run import AsyncScenarioRun
+
+
+class AsyncScenario:
+    """A scenario for evaluating agent performance (async).
+
+    Provides async methods for retrieving scenario details, updating the scenario,
+    and starting scenario runs. Obtain instances via ``runloop.scenario.from_id()``
+    or ``runloop.scenario.list()``.
+
+    Example:
+        >>> scenario = runloop.scenario.from_id("scn-xxx")
+        >>> info = await scenario.get_info()
+        >>> run = await scenario.run(run_name="test-run")
+    """
+
+    def __init__(self, client: AsyncRunloop, scenario_id: str) -> None:
+        """Create an AsyncScenario instance.
+
+        :param client: AsyncRunloop client instance
+        :type client: AsyncRunloop
+        :param scenario_id: Scenario ID
+        :type scenario_id: str
+        """
+        self._client = client
+        self._id = scenario_id
+
+    @override
+    def __repr__(self) -> str:
+        return f"<AsyncScenario id={self._id!r}>"
+
+    @property
+    def id(self) -> str:
+        """Return the scenario ID.
+
+        :return: Unique scenario ID
+        :rtype: str
+        """
+        return self._id
+
+    async def get_info(
+        self,
+        **options: Unpack[BaseRequestOptions],
+    ) -> ScenarioView:
+        """Retrieve current scenario details.
+
+        :param options: See :typeddict:`~runloop_api_client.sdk._types.BaseRequestOptions` for available options
+        :return: Current scenario info
+        :rtype: ScenarioView
+        """
+        return await self._client.scenarios.retrieve(
+            self._id,
+            **options,
+        )
+
+    async def update(
+        self,
+        **params: Unpack[SDKScenarioUpdateParams],
+    ) -> ScenarioView:
+        """Update the scenario.
+
+        Only provided fields will be updated.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScenarioUpdateParams` for available parameters
+        :return: Updated scenario info
+        :rtype: ScenarioView
+        """
+        return await self._client.scenarios.update(
+            self._id,
+            **params,
+        )
+
+    async def run_async(
+        self,
+        **params: Unpack[SDKScenarioRunAsyncParams],
+    ) -> AsyncScenarioRun:
+        """Start a new scenario run without waiting for the devbox.
+
+        Creates a new scenario run and returns immediately. The devbox may still
+        be starting; call ``await_env_ready()`` on the returned AsyncScenarioRun
+        to wait for it to be ready.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScenarioRunAsyncParams` for available parameters
+        :return: AsyncScenarioRun instance for managing the run
+        :rtype: AsyncScenarioRun
+        """
+        run_view = await self._client.scenarios.start_run(
+            scenario_id=self._id,
+            **params,
+        )
+        return AsyncScenarioRun(self._client, run_view.id, run_view.devbox_id)
+
+    async def run(
+        self,
+        **params: Unpack[SDKScenarioRunParams],
+    ) -> AsyncScenarioRun:
+        """Start a new scenario run and wait for the devbox to be ready.
+
+        Convenience method that starts a run and waits for the devbox to be ready.
+
+        :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScenarioRunParams` for available parameters
+        :return: AsyncScenarioRun instance with ready devbox
+        :rtype: AsyncScenarioRun
+        """
+        run_view = await self._client.scenarios.start_run_and_await_env_ready(
+            scenario_id=self._id,
+            **params,
+        )
+        return AsyncScenarioRun(self._client, run_view.id, run_view.devbox_id)