Merge remote-tracking branch 'origin/main' into next

Author: stainless-app[bot]

.gitignore

@@ -10,6 +10,7 @@ dist-deno
 .idea/
 .eslintcache
 .DS_Store
+coverage
 coverage-objects
 object-coverage-report.json
 docs

README.md

@@ -61,13 +61,33 @@ The SDK provides object-oriented interfaces for all major Runloop resources:
 - **[`runloop.blueprint`](https://runloopai.github.io/api-client-ts/stable/classes/BlueprintOps.html)** - Blueprint management (create, list, build blueprints)
 - **[`runloop.snapshot`](https://runloopai.github.io/api-client-ts/stable/classes/SnapshotOps.html)** - Snapshot management (list disk snapshots)
 - **[`runloop.storageObject`](https://runloopai.github.io/api-client-ts/stable/classes/StorageObjectOps.html)** - Storage object management (upload, download, list objects)
+- **[`runloop.agent`](https://runloopai.github.io/api-client-ts/stable/classes/AgentOps.html)** - Agent management (create, list agents from npm/pip/git)
+- **[`runloop.scenario`](https://runloopai.github.io/api-client-ts/stable/classes/ScenarioOps.html)** - Scenario management (list scenarios, start runs)
 - **[`runloop.scorer`](https://runloopai.github.io/api-client-ts/stable/classes/ScorerOps.html)** - Scorer management (create, list, validate, update)
 - **[`runloop.api`](https://runloopai.github.io/api-client-ts/stable/classes/Runloop.html)** - Direct access to the REST API client
 
 ## TypeScript Support
 
 The SDK is fully typed with comprehensive TypeScript definitions:
 
+### Blueprints
+
+Blueprints define reusable devbox configurations. Create blueprints via `runloop.blueprint.create()` and access build logs with `blueprint.logs()`:
+
+```typescript
+const blueprint = await runloop.blueprint.create({
+  name: 'my-blueprint',
+  dockerfile: 'FROM ubuntu:22.04\nRUN apt-get update',
+});
+
+// Get build logs
+const logs = await blueprint.logs();
+console.log(logs.logs);
+
+// Create a devbox from the blueprint
+const devbox = await blueprint.createDevbox({ name: 'my-devbox' });
+```
+
 ### Scorers
 
 Scorers are custom scoring functions used to evaluate scenario outputs. Create scorers via `runloop.scorer.create()`, then update or validate them with the returned `Scorer` instance:
@@ -87,6 +107,37 @@ const result = await scorer.validate({ scoring_context: { output: 'hello' } });
 console.log(result.scoring_result.score);
 ```
 
+### Scenarios
+
+Scenarios define tasks with a well defined starting environment, task evaluation scorer and an optional reference solution.. Use `runloop.scenario.fromId()` to get a scenario, then `scenario.run()` to start a run with your agent mounted:
+
+```typescript
+const scenario = runloop.scenario.fromId('scn_123');
+const run = await scenario.run({
+  run_name: 'my-run',
+  runProfile: {
+    mounts: [
+      {
+        type: 'agent_mount',
+        agent_id: 'agt_123',
+        agent_path: '/home/user/agent',
+      },
+    ],
+  },
+});
+await run.devbox.cmd.exec('python /home/user/agent/main.py');
+await run.scoreAndComplete();
+```
+
+### Benchmarks
+
+Benchmarks are collections of scenarios for evaluating AI agents. Access via `runloop.api.benchmarks`:
+
+```typescript
+const benchmarks = await runloop.api.benchmarks.listPublic();
+const definitions = await runloop.api.benchmarks.definitions('benchmark_id');
+```
+
 ## Migration from API Client
 
 If you're currently using the legacy API, migration is straightforward:

src/sdk.ts

@@ -8,6 +8,7 @@ import { StorageObject } from './sdk/storage-object';
 import { Agent } from './sdk/agent';
 import { Scorer } from './sdk/scorer';
 import { NetworkPolicy } from './sdk/network-policy';
+import { Scenario } from './sdk/scenario';
 
 // Import types used in this file
 import type {
@@ -21,6 +22,7 @@ import type { ObjectCreateParams, ObjectListParams } from './resources/objects';
 import type { AgentCreateParams, AgentListParams } from './resources/agents';
 import type { ScorerCreateParams, ScorerListParams } from './resources/scenarios/scorers';
 import type { NetworkPolicyCreateParams, NetworkPolicyListParams } from './resources/network-policies';
+import type { ScenarioListParams } from './resources/scenarios/scenarios';
 import { PollingOptions } from './lib/polling';
 import * as Shared from './resources/shared';
 
@@ -282,6 +284,14 @@ export class RunloopSDK {
    */
   public readonly networkPolicy: NetworkPolicyOps;
 
+  /**
+   * **Scenario Operations** - {@link ScenarioOps} for accessing {@link Scenario} class instances.
+   *
+   * Scenarios define repeatable AI coding evaluation tests with starting environments and
+   * success criteria. Use these operations to get existing scenarios by ID or list all scenarios.
+   */
+  public readonly scenario: ScenarioOps;
+
   /**
    * Creates a new RunloopSDK instance.
    * @param {ClientOptions} [options] - Optional client configuration options.
@@ -295,6 +305,7 @@ export class RunloopSDK {
     this.agent = new AgentOps(this.api);
     this.scorer = new ScorerOps(this.api);
     this.networkPolicy = new NetworkPolicyOps(this.api);
+    this.scenario = new ScenarioOps(this.api);
   }
 }
 
@@ -433,7 +444,7 @@ export class DevboxOps {
   }
 
   /**
-   * List all devboxes with optional filters.
+   * List devboxes with optional filters (paginated).
    * @param {DevboxListParams} [params] - Optional filter parameters.
    * @param {Core.RequestOptions} [options] - Request options.
    * @returns {Promise<Devbox[]>} An array of {@link Devbox} instances.
@@ -529,7 +540,7 @@ export class BlueprintOps {
   }
 
   /**
-   * List all blueprints with optional filters.
+   * List blueprints with optional filters (paginated).
    * @param {BlueprintListParams} [params] - Optional filter parameters.
    * @param {Core.RequestOptions} [options] - Request options.
    * @returns {Promise<Blueprint[]>} An array of {@link Blueprint} instances.
@@ -587,7 +598,7 @@ export class SnapshotOps {
   }
 
   /**
-   * List all snapshots.
+   * List snapshots with optional filters (paginated).
    * @param {DevboxListDiskSnapshotsParams} [params] - Optional filter parameters.
    * @param {Core.RequestOptions} [options] - Request options.
    * @returns {Promise<Snapshot[]>} An array of {@link Snapshot} instances.
@@ -663,7 +674,7 @@ export class StorageObjectOps {
   }
 
   /**
-   * List all storage objects with optional filters.
+   * List storage objects with optional filters (paginated).
    * @param {ObjectListParams} [params] - Optional filter parameters.
    * @param {Core.RequestOptions} [options] - Request options.
    * @returns {Promise<StorageObject[]>} An array of {@link StorageObject} instances.
@@ -1246,7 +1257,7 @@ export class AgentOps {
   }
 
   /**
-   * List all agents with optional filters.
+   * List agents with optional filters (paginated).
    *
    * @example
    * List all agents:
@@ -1376,7 +1387,7 @@ export class ScorerOps {
   }
 
   /**
-   * List all scorers with optional filters.
+   * List scorers with optional filters (paginated).
    *
    * @example
    * ```typescript
@@ -1472,7 +1483,7 @@ export class NetworkPolicyOps {
   }
 
   /**
-   * List all network policies with optional filters.
+   * List network policies with optional filters (paginated).
    *
    * @example
    * ```typescript
@@ -1497,6 +1508,106 @@ export class NetworkPolicyOps {
   }
 }
 
+/**
+ * Scenario SDK interface for managing scenarios.
+ *
+ * @category Scenario
+ *
+ * @remarks
+ * ## Overview
+ *
+ * The `ScenarioOps` class provides a high-level abstraction for managing scenarios.
+ *
+ * ## Quickstart
+ *
+ * Use `fromId()` to get a {@link Scenario} by ID, or `list()` to retrieve all scenarios.
+ * Once you have a scenario, call `scenario.run()` to start a {@link ScenarioRun} with
+ * your agent mounted.
+ *
+ * ## Usage
+ *
+ * This interface is accessed via {@link RunloopSDK.scenario}. You should construct
+ * a {@link RunloopSDK} instance and use it from there:
+ *
+ * @example
+ * ```typescript
+ * const runloop = new RunloopSDK();
+ * const scenario = runloop.scenario.fromId('scn_123');
+ *
+ * // Get scenario details
+ * const info = await scenario.getInfo();
+ * console.log(info.name);
+ *
+ * // Start a run with agent mounted and wait for the devbox to be ready
+ * const run = await scenario.run({
+ *   run_name: 'my-run',
+ *   runProfile: {
+ *     mounts: [{
+ *       type: 'agent_mount',
+ *       agent_id: 'agt_123',
+ *       agent_name: null,
+ *       agent_path: '/home/user/agent',
+ *     }],
+ *   },
+ * });
+ *
+ * // Execute your agent on the devbox
+ * await run.devbox.cmd.exec('python /home/user/agent/main.py');
+ *
+ * // Score and complete
+ * await run.scoreAndComplete();
+ * ```
+ */
+export class ScenarioOps {
+  /**
+   * @private
+   */
+  constructor(private client: RunloopAPI) {}
+
+  /**
+   * Get a scenario object by its ID.
+   *
+   * @example
+   * ```typescript
+   * const runloop = new RunloopSDK();
+   * const scenario = runloop.scenario.fromId('scn_123');
+   * const info = await scenario.getInfo();
+   * console.log(info.name);
+   * ```
+   *
+   * @param {string} id - The ID of the scenario
+   * @returns {Scenario} A {@link Scenario} instance
+   */
+  fromId(id: string): Scenario {
+    return Scenario.fromId(this.client, id);
+  }
+
+  /**
+   * List scenarios with optional filters (paginated).
+   *
+   * @example
+   * ```typescript
+   * const runloop = new RunloopSDK();
+   * const scenarios = await runloop.scenario.list({ limit: 10 });
+   * console.log(scenarios.map((s) => s.id));
+   * ```
+   *
+   * @param {ScenarioListParams} [params] - Optional filter parameters
+   * @param {Core.RequestOptions} [options] - Request options
+   * @returns {Promise<Scenario[]>} An array of {@link Scenario} instances
+   */
+  async list(params?: ScenarioListParams, options?: Core.RequestOptions): Promise<Scenario[]> {
+    const result = await this.client.scenarios.list(params, options);
+    const scenarios: Scenario[] = [];
+
+    for (const scenario of result.scenarios) {
+      scenarios.push(Scenario.fromId(this.client, scenario.id));
+    }
+
+    return scenarios;
+  }
+}
+
 // @deprecated Use {@link RunloopSDK} instead.
 /**
  * @deprecated Use {@link RunloopSDK} instead.
@@ -1519,13 +1630,15 @@ export declare namespace RunloopSDK {
     AgentOps as AgentOps,
     ScorerOps as ScorerOps,
     NetworkPolicyOps as NetworkPolicyOps,
+    ScenarioOps as ScenarioOps,
     Devbox as Devbox,
     Blueprint as Blueprint,
     Snapshot as Snapshot,
     StorageObject as StorageObject,
     Agent as Agent,
     Scorer as Scorer,
     NetworkPolicy as NetworkPolicy,
+    Scenario as Scenario,
   };
 }
 // Export SDK classes from sdk/sdk.ts - these are separate from RunloopSDK to avoid circular dependencies
@@ -1543,4 +1656,6 @@ export {
   NetworkPolicy,
   Execution,
   ExecutionResult,
+  Scenario,
+  ScenarioRun,
 } from './sdk/index';

src/sdk/index.ts

@@ -8,3 +8,4 @@ export { ExecutionResult } from './execution-result';
 export { Scorer } from './scorer';
 export { NetworkPolicy } from './network-policy';
 export { ScenarioRun } from './scenario-run';
+export { Scenario, type ScenarioRunParams } from './scenario';

src/sdk/scenario-run.ts

@@ -27,12 +27,23 @@ import * as path from 'path';
  * import { RunloopSDK } from '@runloop/api-client';
  *
  * const runloop = new RunloopSDK();
- * const scenario = runloop.scenario.fromId('scenario-123');
- * const run = await scenario.run({ run_name: 'my-run' });
+ * const scenario = runloop.scenario.fromId('scn_123');
  *
- * // Access the devbox and execute your agent to solve the scenario
- * const devbox = run.devbox;
- * await devbox.cmd.exec('python /home/user/agent/main.py');
+ * // Start a run with agent mounted
+ * const run = await scenario.run({
+ *   run_name: 'my-run',
+ *   runProfile: {
+ *     mounts: [{
+ *       type: 'agent_mount',
+ *       agent_id: 'agt_123',
+ *       agent_name: null,
+ *       agent_path: '/home/user/agent',
+ *     }],
+ *   },
+ * });
+ *
+ * // Execute your agent on the devbox
+ * await run.devbox.cmd.exec('python /home/user/agent/main.py');
  *
  * // Score and complete the run
  * await run.scoreAndComplete();