fix(benchmarks): update() for benchmarks and scenarios replaces all provided fields and does not modify unspecified fields (#6702)

Author: stainless-app[bot]

.stats.yml

@@ -1,4 +1,4 @@
 configured_endpoints: 98
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-370a5d676ff0ef41f3ca6ed669d29b6e3e6b7f0a914ac3dbed6bd45e75533c74.yml
-openapi_spec_hash: 7bb5ea1117d754b7985aff0da5bca3a7
-config_hash: 7b7fc2ccd6bf3551718a79332772e713
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-939601ae2016f4611626f9b52a14a4b4a2076879f49804942ac1788b3302b97a.yml
+openapi_spec_hash: 162edc35b9e7b1f45fe38484361aa880
+config_hash: cb8534d20a68a49b92726bedd50f8bb1

api.md

@@ -17,6 +17,7 @@ Types:
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkRunListView</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkRunView</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkScenarioUpdateParameters</a></code>
+- <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkUpdateParameters</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">BenchmarkView</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">ScenarioDefinitionListView</a></code>
 - <code><a href="./src/resources/benchmarks/benchmarks.ts">StartBenchmarkRunParameters</a></code>

src/index.ts

@@ -108,6 +108,7 @@ import {
   BenchmarkRunView,
   BenchmarkScenarioUpdateParameters,
   BenchmarkStartRunParams,
+  BenchmarkUpdateParameters,
   BenchmarkUpdateParams,
   BenchmarkUpdateScenariosParams,
   BenchmarkView,
@@ -445,6 +446,7 @@ export declare namespace Runloop {
     type BenchmarkRunListView as BenchmarkRunListView,
     type BenchmarkRunView as BenchmarkRunView,
     type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters,
+    type BenchmarkUpdateParameters as BenchmarkUpdateParameters,
     type BenchmarkView as BenchmarkView,
     type ScenarioDefinitionListView as ScenarioDefinitionListView,
     type StartBenchmarkRunParameters as StartBenchmarkRunParameters,

src/resources/benchmarks/benchmarks.ts

@@ -31,13 +31,24 @@ export class Benchmarks extends APIResource {
   }
 
   /**
-   * Update a Benchmark with a set of Scenarios.
+   * Update a Benchmark. Fields that are null will preserve the existing value.
+   * Fields that are provided (including empty values) will replace the existing
+   * value entirely.
    */
   update(
     id: string,
-    body: BenchmarkUpdateParams,
+    body?: BenchmarkUpdateParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<BenchmarkView>;
+  update(id: string, options?: Core.RequestOptions): Core.APIPromise<BenchmarkView>;
+  update(
+    id: string,
+    body: BenchmarkUpdateParams | Core.RequestOptions = {},
     options?: Core.RequestOptions,
   ): Core.APIPromise<BenchmarkView> {
+    if (isRequestOptions(body)) {
+      return this.update(id, {}, body);
+    }
     return this._client.post(`/v1/benchmarks/${id}`, { body, ...options });
   }
 
@@ -142,7 +153,7 @@ export class BenchmarkRunViewsBenchmarkRunsCursorIDPage extends BenchmarkRunsCur
  */
 export interface BenchmarkCreateParameters {
   /**
-   * The name of the Benchmark. This must be unique.
+   * The unique name of the Benchmark.
    */
   name: string;
 
@@ -157,13 +168,13 @@ export interface BenchmarkCreateParameters {
   description?: string | null;
 
   /**
-   * User defined metadata to attach to the benchmark for organization.
+   * User defined metadata to attach to the benchmark.
    */
   metadata?: { [key: string]: string } | null;
 
   /**
    * Environment variables required to run the benchmark. If any required variables
-   * are not supplied, the benchmark will fail to start
+   * are not supplied, the benchmark will fail to start.
    */
   required_environment_variables?: Array<string> | null;
 
@@ -269,6 +280,52 @@ export interface BenchmarkScenarioUpdateParameters {
   scenarios_to_remove?: Array<string> | null;
 }
 
+/**
+ * BenchmarkUpdateParameters contain the set of parameters to update a Benchmark.
+ * All fields are optional - null fields preserve existing values, provided fields
+ * replace entirely.
+ */
+export interface BenchmarkUpdateParameters {
+  /**
+   * Attribution information for the benchmark. Pass in empty string to clear.
+   */
+  attribution?: string | null;
+
+  /**
+   * Detailed description of the benchmark. Pass in empty string to clear.
+   */
+  description?: string | null;
+
+  /**
+   * User defined metadata to attach to the benchmark. Pass in empty map to clear.
+   */
+  metadata?: { [key: string]: string } | null;
+
+  /**
+   * The unique name of the Benchmark. Cannot be blank.
+   */
+  name?: string | null;
+
+  /**
+   * Environment variables required to run the benchmark. If any required variables
+   * are not supplied, the benchmark will fail to start. Pass in empty list to clear.
+   */
+  required_environment_variables?: Array<string> | null;
+
+  /**
+   * Secrets required to run the benchmark with (environment variable name will be
+   * mapped to the your user secret by name). If any of these secrets are not
+   * provided or the mapping is incorrect, the benchmark will fail to start. Pass in
+   * empty list to clear.
+   */
+  required_secret_names?: Array<string> | null;
+
+  /**
+   * The Scenario IDs that make up the Benchmark. Pass in empty list to clear.
+   */
+  scenario_ids?: Array<string> | null;
+}
+
 /**
  * A BenchmarkDefinitionView represents a grouped set of Scenarios that together
  * form a Benchmark.
@@ -359,7 +416,7 @@ export interface StartBenchmarkRunParameters {
 
 export interface BenchmarkCreateParams {
   /**
-   * The name of the Benchmark. This must be unique.
+   * The unique name of the Benchmark.
    */
   name: string;
 
@@ -374,13 +431,13 @@ export interface BenchmarkCreateParams {
   description?: string | null;
 
   /**
-   * User defined metadata to attach to the benchmark for organization.
+   * User defined metadata to attach to the benchmark.
    */
   metadata?: { [key: string]: string } | null;
 
   /**
    * Environment variables required to run the benchmark. If any required variables
-   * are not supplied, the benchmark will fail to start
+   * are not supplied, the benchmark will fail to start.
    */
   required_environment_variables?: Array<string> | null;
 
@@ -399,40 +456,41 @@ export interface BenchmarkCreateParams {
 
 export interface BenchmarkUpdateParams {
   /**
-   * The name of the Benchmark. This must be unique.
-   */
-  name: string;
-
-  /**
-   * Attribution information for the benchmark.
+   * Attribution information for the benchmark. Pass in empty string to clear.
    */
   attribution?: string | null;
 
   /**
-   * Detailed description of the benchmark.
+   * Detailed description of the benchmark. Pass in empty string to clear.
    */
   description?: string | null;
 
   /**
-   * User defined metadata to attach to the benchmark for organization.
+   * User defined metadata to attach to the benchmark. Pass in empty map to clear.
    */
   metadata?: { [key: string]: string } | null;
 
+  /**
+   * The unique name of the Benchmark. Cannot be blank.
+   */
+  name?: string | null;
+
   /**
    * Environment variables required to run the benchmark. If any required variables
-   * are not supplied, the benchmark will fail to start
+   * are not supplied, the benchmark will fail to start. Pass in empty list to clear.
    */
   required_environment_variables?: Array<string> | null;
 
   /**
    * Secrets required to run the benchmark with (environment variable name will be
    * mapped to the your user secret by name). If any of these secrets are not
-   * provided or the mapping is incorrect, the benchmark will fail to start.
+   * provided or the mapping is incorrect, the benchmark will fail to start. Pass in
+   * empty list to clear.
    */
-  required_secret_names?: Array<string>;
+  required_secret_names?: Array<string> | null;
 
   /**
-   * The Scenario IDs that make up the Benchmark.
+   * The Scenario IDs that make up the Benchmark. Pass in empty list to clear.
    */
   scenario_ids?: Array<string> | null;
 }
@@ -501,6 +559,7 @@ export declare namespace Benchmarks {
     type BenchmarkRunListView as BenchmarkRunListView,
     type BenchmarkRunView as BenchmarkRunView,
     type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters,
+    type BenchmarkUpdateParameters as BenchmarkUpdateParameters,
     type BenchmarkView as BenchmarkView,
     type ScenarioDefinitionListView as ScenarioDefinitionListView,
     type StartBenchmarkRunParameters as StartBenchmarkRunParameters,

src/resources/benchmarks/index.ts

@@ -8,6 +8,7 @@ export {
   type BenchmarkRunListView,
   type BenchmarkRunView,
   type BenchmarkScenarioUpdateParameters,
+  type BenchmarkUpdateParameters,
   type BenchmarkView,
   type ScenarioDefinitionListView,
   type StartBenchmarkRunParameters,

src/resources/index.ts

@@ -18,6 +18,7 @@ export {
   type BenchmarkRunListView,
   type BenchmarkRunView,
   type BenchmarkScenarioUpdateParameters,
+  type BenchmarkUpdateParameters,
   type BenchmarkView,
   type ScenarioDefinitionListView,
   type StartBenchmarkRunParameters,

src/resources/scenarios/scenarios.ts

@@ -48,9 +48,9 @@ export class Scenarios extends APIResource {
   }
 
   /**
-   * Update a Scenario, a repeatable AI coding evaluation test that defines the
-   * starting environment as well as evaluation success criteria. Only provided
-   * fields will be updated.
+   * Update a Scenario. Fields that are null will preserve the existing value. Fields
+   * that are provided (including empty values) will replace the existing value
+   * entirely.
    */
   update(
     id: string,
@@ -329,6 +329,11 @@ export interface ScenarioRunView {
   start_time_ms?: number;
 }
 
+/**
+ * ScenarioUpdateParameters contain the set of parameters to update a Scenario. All
+ * fields are optional - null fields preserve existing values, provided fields
+ * replace entirely.
+ */
 export interface ScenarioUpdateParameters {
   /**
    * The Environment in which the Scenario will run.
@@ -341,29 +346,29 @@ export interface ScenarioUpdateParameters {
   input_context?: InputContextUpdate | null;
 
   /**
-   * User defined metadata to attach to the scenario for organization.
+   * User defined metadata to attach to the scenario. Pass in empty map to clear.
    */
   metadata?: { [key: string]: string } | null;
 
   /**
-   * Name of the scenario.
+   * Name of the scenario. Cannot be blank.
    */
   name?: string | null;
 
   /**
    * A string representation of the reference output to solve the scenario. Commonly
    * can be the result of a git diff or a sequence of command actions to apply to the
-   * environment.
+   * environment. Pass in empty string to clear.
    */
   reference_output?: string | null;
 
   /**
-   * Environment variables required to run the scenario.
+   * Environment variables required to run the scenario. Pass in empty list to clear.
    */
   required_environment_variables?: Array<string> | null;
 
   /**
-   * Secrets required to run the scenario.
+   * Secrets required to run the scenario. Pass in empty list to clear.
    */
   required_secret_names?: Array<string> | null;
 
@@ -373,7 +378,7 @@ export interface ScenarioUpdateParameters {
   scoring_contract?: ScoringContractUpdate | null;
 
   /**
-   * Validation strategy.
+   * Validation strategy. Pass in empty string to clear.
    */
   validation_type?: 'UNSPECIFIED' | 'FORWARD' | 'REVERSE' | 'EVALUATION' | null;
 }
@@ -752,29 +757,29 @@ export interface ScenarioUpdateParams {
   input_context?: InputContextUpdate | null;
 
   /**
-   * User defined metadata to attach to the scenario for organization.
+   * User defined metadata to attach to the scenario. Pass in empty map to clear.
    */
   metadata?: { [key: string]: string } | null;
 
   /**
-   * Name of the scenario.
+   * Name of the scenario. Cannot be blank.
    */
   name?: string | null;
 
   /**
    * A string representation of the reference output to solve the scenario. Commonly
    * can be the result of a git diff or a sequence of command actions to apply to the
-   * environment.
+   * environment. Pass in empty string to clear.
    */
   reference_output?: string | null;
 
   /**
-   * Environment variables required to run the scenario.
+   * Environment variables required to run the scenario. Pass in empty list to clear.
    */
   required_environment_variables?: Array<string> | null;
 
   /**
-   * Secrets required to run the scenario.
+   * Secrets required to run the scenario. Pass in empty list to clear.
    */
   required_secret_names?: Array<string> | null;
 
@@ -784,7 +789,7 @@ export interface ScenarioUpdateParams {
   scoring_contract?: ScoringContractUpdate | null;
 
   /**
-   * Validation strategy.
+   * Validation strategy. Pass in empty string to clear.
    */
   validation_type?: 'UNSPECIFIED' | 'FORWARD' | 'REVERSE' | 'EVALUATION' | null;
 }