feat(benchmarks): add update_scenarios method to benchmarks resource

Author: stainless-app[bot]

.stats.yml

@@ -1,4 +1,4 @@
-configured_endpoints: 97
+configured_endpoints: 98
 openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-370a5d676ff0ef41f3ca6ed669d29b6e3e6b7f0a914ac3dbed6bd45e75533c74.yml
 openapi_spec_hash: 7bb5ea1117d754b7985aff0da5bca3a7
-config_hash: 2363f563f42501d2b1587a4f64bdccaf
+config_hash: 7b7fc2ccd6bf3551718a79332772e713

api.md

@@ -16,6 +16,7 @@ Types:
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkCreateParameters</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkRunListView</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkRunView</a></code>
+- <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkScenarioUpdateParameters</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkView</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">ScenarioDefinitionListView</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">StartBenchmarkRunParameters</a></code>
@@ -29,6 +30,7 @@ Methods:
 - <code title="get /v1/benchmarks/{id}/definitions">client.benchmarks.<a href="./src/resources/benchmarks/benchmarks.ts">definitions</a>(id, { ...params }) -> ScenarioDefinitionListView</code>
 - <code title="get /v1/benchmarks/list_public">client.benchmarks.<a href="./src/resources/benchmarks/benchmarks.ts">listPublic</a>({ ...params }) -> BenchmarkViewsBenchmarksCursorIDPage</code>
 - <code title="post /v1/benchmarks/start_run">client.benchmarks.<a href="./src/resources/benchmarks/benchmarks.ts">startRun</a>({ ...params }) -> BenchmarkRunView</code>
+- <code title="post /v1/benchmarks/{id}/scenarios">client.benchmarks.<a href="./src/resources/benchmarks/benchmarks.ts">updateScenarios</a>(id, { ...params }) -> BenchmarkView</code>
 
 ## Runs
 

src/index.ts

@@ -106,8 +106,10 @@ import {
   BenchmarkListPublicParams,
   BenchmarkRunListView,
   BenchmarkRunView,
+  BenchmarkScenarioUpdateParameters,
   BenchmarkStartRunParams,
   BenchmarkUpdateParams,
+  BenchmarkUpdateScenariosParams,
   BenchmarkView,
   BenchmarkViewsBenchmarksCursorIDPage,
   Benchmarks,
@@ -442,6 +444,7 @@ export declare namespace Runloop {
     type BenchmarkCreateParameters as BenchmarkCreateParameters,
     type BenchmarkRunListView as BenchmarkRunListView,
     type BenchmarkRunView as BenchmarkRunView,
+    type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters,
     type BenchmarkView as BenchmarkView,
     type ScenarioDefinitionListView as ScenarioDefinitionListView,
     type StartBenchmarkRunParameters as StartBenchmarkRunParameters,
@@ -452,6 +455,7 @@ export declare namespace Runloop {
     type BenchmarkDefinitionsParams as BenchmarkDefinitionsParams,
     type BenchmarkListPublicParams as BenchmarkListPublicParams,
     type BenchmarkStartRunParams as BenchmarkStartRunParams,
+    type BenchmarkUpdateScenariosParams as BenchmarkUpdateScenariosParams,
   };
 
   export {

src/resources/benchmarks/benchmarks.ts

@@ -111,6 +111,26 @@ export class Benchmarks extends APIResource {
   startRun(body: BenchmarkStartRunParams, options?: Core.RequestOptions): Core.APIPromise<BenchmarkRunView> {
     return this._client.post('/v1/benchmarks/start_run', { body, ...options });
   }
+
+  /**
+   * Add and/or remove Scenario IDs from an existing Benchmark.
+   */
+  updateScenarios(
+    id: string,
+    body?: BenchmarkUpdateScenariosParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<BenchmarkView>;
+  updateScenarios(id: string, options?: Core.RequestOptions): Core.APIPromise<BenchmarkView>;
+  updateScenarios(
+    id: string,
+    body: BenchmarkUpdateScenariosParams | Core.RequestOptions = {},
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<BenchmarkView> {
+    if (isRequestOptions(body)) {
+      return this.updateScenarios(id, {}, body);
+    }
+    return this._client.post(`/v1/benchmarks/${id}/scenarios`, { body, ...options });
+  }
 }
 
 export class BenchmarkViewsBenchmarksCursorIDPage extends BenchmarksCursorIDPage<BenchmarkView> {}
@@ -237,6 +257,18 @@ export interface BenchmarkRunView {
   secrets_provided?: { [key: string]: string } | null;
 }
 
+export interface BenchmarkScenarioUpdateParameters {
+  /**
+   * Scenario IDs to add to the Benchmark.
+   */
+  scenarios_to_add?: Array<string> | null;
+
+  /**
+   * Scenario IDs to remove from the Benchmark.
+   */
+  scenarios_to_remove?: Array<string> | null;
+}
+
 /**
  * A BenchmarkDefinitionView represents a grouped set of Scenarios that together
  * form a Benchmark.
@@ -448,6 +480,18 @@ export interface BenchmarkStartRunParams {
   runProfile?: Shared.RunProfile | null;
 }
 
+export interface BenchmarkUpdateScenariosParams {
+  /**
+   * Scenario IDs to add to the Benchmark.
+   */
+  scenarios_to_add?: Array<string> | null;
+
+  /**
+   * Scenario IDs to remove from the Benchmark.
+   */
+  scenarios_to_remove?: Array<string> | null;
+}
+
 Benchmarks.BenchmarkViewsBenchmarksCursorIDPage = BenchmarkViewsBenchmarksCursorIDPage;
 Benchmarks.Runs = Runs;
 
@@ -456,6 +500,7 @@ export declare namespace Benchmarks {
     type BenchmarkCreateParameters as BenchmarkCreateParameters,
     type BenchmarkRunListView as BenchmarkRunListView,
     type BenchmarkRunView as BenchmarkRunView,
+    type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters,
     type BenchmarkView as BenchmarkView,
     type ScenarioDefinitionListView as ScenarioDefinitionListView,
     type StartBenchmarkRunParameters as StartBenchmarkRunParameters,
@@ -466,6 +511,7 @@ export declare namespace Benchmarks {
     type BenchmarkDefinitionsParams as BenchmarkDefinitionsParams,
     type BenchmarkListPublicParams as BenchmarkListPublicParams,
     type BenchmarkStartRunParams as BenchmarkStartRunParams,
+    type BenchmarkUpdateScenariosParams as BenchmarkUpdateScenariosParams,
   };
 
   export {

src/resources/benchmarks/index.ts

@@ -7,6 +7,7 @@ export {
   type BenchmarkCreateParameters,
   type BenchmarkRunListView,
   type BenchmarkRunView,
+  type BenchmarkScenarioUpdateParameters,
   type BenchmarkView,
   type ScenarioDefinitionListView,
   type StartBenchmarkRunParameters,
@@ -16,5 +17,6 @@ export {
   type BenchmarkDefinitionsParams,
   type BenchmarkListPublicParams,
   type BenchmarkStartRunParams,
+  type BenchmarkUpdateScenariosParams,
 } from './benchmarks';
 export { Runs, type RunListParams, type RunListScenarioRunsParams } from './runs';

src/resources/index.ts

@@ -17,6 +17,7 @@ export {
   type BenchmarkCreateParameters,
   type BenchmarkRunListView,
   type BenchmarkRunView,
+  type BenchmarkScenarioUpdateParameters,
   type BenchmarkView,
   type ScenarioDefinitionListView,
   type StartBenchmarkRunParameters,
@@ -26,6 +27,7 @@ export {
   type BenchmarkDefinitionsParams,
   type BenchmarkListPublicParams,
   type BenchmarkStartRunParams,
+  type BenchmarkUpdateScenariosParams,
 } from './benchmarks/benchmarks';
 export {
   BlueprintViewsBlueprintsCursorIDPage,

tests/api-resources/benchmarks/benchmarks.test.ts

@@ -195,4 +195,33 @@ describe('resource benchmarks', () => {
       },
     });
   });
+
+  test('updateScenarios', async () => {
+    const responsePromise = client.benchmarks.updateScenarios('id');
+    const rawResponse = await responsePromise.asResponse();
+    expect(rawResponse).toBeInstanceOf(Response);
+    const response = await responsePromise;
+    expect(response).not.toBeInstanceOf(Response);
+    const dataAndResponse = await responsePromise.withResponse();
+    expect(dataAndResponse.data).toBe(response);
+    expect(dataAndResponse.response).toBe(rawResponse);
+  });
+
+  test('updateScenarios: request options instead of params are passed correctly', async () => {
+    // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error
+    await expect(
+      client.benchmarks.updateScenarios('id', { path: '/_stainless_unknown_path' }),
+    ).rejects.toThrow(Runloop.NotFoundError);
+  });
+
+  test('updateScenarios: request options and params are passed correctly', async () => {
+    // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error
+    await expect(
+      client.benchmarks.updateScenarios(
+        'id',
+        { scenarios_to_add: ['string'], scenarios_to_remove: ['string'] },
+        { path: '/_stainless_unknown_path' },
+      ),
+    ).rejects.toThrow(Runloop.NotFoundError);
+  });
 });