fix(benchmarks): update() for benchmarks and scenarios replaces all provided fields and does not modify unspecified fields (#6702)

Author: stainless-app[bot]

.stats.yml

@@ -1,4 +1,4 @@
 configured_endpoints: 98
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-370a5d676ff0ef41f3ca6ed669d29b6e3e6b7f0a914ac3dbed6bd45e75533c74.yml
-openapi_spec_hash: 7bb5ea1117d754b7985aff0da5bca3a7
-config_hash: 7b7fc2ccd6bf3551718a79332772e713
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-939601ae2016f4611626f9b52a14a4b4a2076879f49804942ac1788b3302b97a.yml
+openapi_spec_hash: 162edc35b9e7b1f45fe38484361aa880
+config_hash: cb8534d20a68a49b92726bedd50f8bb1

api.md

@@ -21,6 +21,7 @@ from runloop_api_client.types import (
     BenchmarkRunListView,
     BenchmarkRunView,
     BenchmarkScenarioUpdateParameters,
+    BenchmarkUpdateParameters,
     BenchmarkView,
     ScenarioDefinitionListView,
     StartBenchmarkRunParameters,

src/runloop_api_client/resources/benchmarks/benchmarks.py

@@ -89,16 +89,16 @@ def create(
         Create a Benchmark with a set of Scenarios.
 
         Args:
-          name: The name of the Benchmark. This must be unique.
+          name: The unique name of the Benchmark.
 
           attribution: Attribution information for the benchmark.
 
           description: Detailed description of the benchmark.
 
-          metadata: User defined metadata to attach to the benchmark for organization.
+          metadata: User defined metadata to attach to the benchmark.
 
           required_environment_variables: Environment variables required to run the benchmark. If any required variables
-              are not supplied, the benchmark will fail to start
+              are not supplied, the benchmark will fail to start.
 
           required_secret_names: Secrets required to run the benchmark with (environment variable name will be
               mapped to the your user secret by name). If any of these secrets are not
@@ -177,12 +177,12 @@ def update(
         self,
         id: str,
         *,
-        name: str,
         attribution: Optional[str] | Omit = omit,
         description: Optional[str] | Omit = omit,
         metadata: Optional[Dict[str, str]] | Omit = omit,
+        name: Optional[str] | Omit = omit,
         required_environment_variables: Optional[SequenceNotStr[str]] | Omit = omit,
-        required_secret_names: SequenceNotStr[str] | Omit = omit,
+        required_secret_names: Optional[SequenceNotStr[str]] | Omit = omit,
         scenario_ids: Optional[SequenceNotStr[str]] | Omit = omit,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -192,26 +192,30 @@ def update(
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
         idempotency_key: str | None = None,
     ) -> BenchmarkView:
-        """
-        Update a Benchmark with a set of Scenarios.
+        """Update a Benchmark.
+
+        Fields that are null will preserve the existing value.
+        Fields that are provided (including empty values) will replace the existing
+        value entirely.
 
         Args:
-          name: The name of the Benchmark. This must be unique.
+          attribution: Attribution information for the benchmark. Pass in empty string to clear.
 
-          attribution: Attribution information for the benchmark.
+          description: Detailed description of the benchmark. Pass in empty string to clear.
 
-          description: Detailed description of the benchmark.
+          metadata: User defined metadata to attach to the benchmark. Pass in empty map to clear.
 
-          metadata: User defined metadata to attach to the benchmark for organization.
+          name: The unique name of the Benchmark. Cannot be blank.
 
           required_environment_variables: Environment variables required to run the benchmark. If any required variables
-              are not supplied, the benchmark will fail to start
+              are not supplied, the benchmark will fail to start. Pass in empty list to clear.
 
           required_secret_names: Secrets required to run the benchmark with (environment variable name will be
               mapped to the your user secret by name). If any of these secrets are not
-              provided or the mapping is incorrect, the benchmark will fail to start.
+              provided or the mapping is incorrect, the benchmark will fail to start. Pass in
+              empty list to clear.
 
-          scenario_ids: The Scenario IDs that make up the Benchmark.
+          scenario_ids: The Scenario IDs that make up the Benchmark. Pass in empty list to clear.
 
           extra_headers: Send extra headers
 
@@ -229,10 +233,10 @@ def update(
             f"/v1/benchmarks/{id}",
             body=maybe_transform(
                 {
-                    "name": name,
                     "attribution": attribution,
                     "description": description,
                     "metadata": metadata,
+                    "name": name,
                     "required_environment_variables": required_environment_variables,
                     "required_secret_names": required_secret_names,
                     "scenario_ids": scenario_ids,
@@ -554,16 +558,16 @@ async def create(
         Create a Benchmark with a set of Scenarios.
 
         Args:
-          name: The name of the Benchmark. This must be unique.
+          name: The unique name of the Benchmark.
 
           attribution: Attribution information for the benchmark.
 
           description: Detailed description of the benchmark.
 
-          metadata: User defined metadata to attach to the benchmark for organization.
+          metadata: User defined metadata to attach to the benchmark.
 
           required_environment_variables: Environment variables required to run the benchmark. If any required variables
-              are not supplied, the benchmark will fail to start
+              are not supplied, the benchmark will fail to start.
 
           required_secret_names: Secrets required to run the benchmark with (environment variable name will be
               mapped to the your user secret by name). If any of these secrets are not
@@ -642,12 +646,12 @@ async def update(
         self,
         id: str,
         *,
-        name: str,
         attribution: Optional[str] | Omit = omit,
         description: Optional[str] | Omit = omit,
         metadata: Optional[Dict[str, str]] | Omit = omit,
+        name: Optional[str] | Omit = omit,
         required_environment_variables: Optional[SequenceNotStr[str]] | Omit = omit,
-        required_secret_names: SequenceNotStr[str] | Omit = omit,
+        required_secret_names: Optional[SequenceNotStr[str]] | Omit = omit,
         scenario_ids: Optional[SequenceNotStr[str]] | Omit = omit,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -657,26 +661,30 @@ async def update(
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
         idempotency_key: str | None = None,
     ) -> BenchmarkView:
-        """
-        Update a Benchmark with a set of Scenarios.
+        """Update a Benchmark.
+
+        Fields that are null will preserve the existing value.
+        Fields that are provided (including empty values) will replace the existing
+        value entirely.
 
         Args:
-          name: The name of the Benchmark. This must be unique.
+          attribution: Attribution information for the benchmark. Pass in empty string to clear.
 
-          attribution: Attribution information for the benchmark.
+          description: Detailed description of the benchmark. Pass in empty string to clear.
 
-          description: Detailed description of the benchmark.
+          metadata: User defined metadata to attach to the benchmark. Pass in empty map to clear.
 
-          metadata: User defined metadata to attach to the benchmark for organization.
+          name: The unique name of the Benchmark. Cannot be blank.
 
           required_environment_variables: Environment variables required to run the benchmark. If any required variables
-              are not supplied, the benchmark will fail to start
+              are not supplied, the benchmark will fail to start. Pass in empty list to clear.
 
           required_secret_names: Secrets required to run the benchmark with (environment variable name will be
               mapped to the your user secret by name). If any of these secrets are not
-              provided or the mapping is incorrect, the benchmark will fail to start.
+              provided or the mapping is incorrect, the benchmark will fail to start. Pass in
+              empty list to clear.
 
-          scenario_ids: The Scenario IDs that make up the Benchmark.
+          scenario_ids: The Scenario IDs that make up the Benchmark. Pass in empty list to clear.
 
           extra_headers: Send extra headers
 
@@ -694,10 +702,10 @@ async def update(
             f"/v1/benchmarks/{id}",
             body=await async_maybe_transform(
                 {
-                    "name": name,
                     "attribution": attribution,
                     "description": description,
                     "metadata": metadata,
+                    "name": name,
                     "required_environment_variables": required_environment_variables,
                     "required_secret_names": required_secret_names,
                     "scenario_ids": scenario_ids,

src/runloop_api_client/resources/scenarios/scenarios.py

@@ -221,31 +221,32 @@ def update(
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
         idempotency_key: str | None = None,
     ) -> ScenarioView:
-        """
-        Update a Scenario, a repeatable AI coding evaluation test that defines the
-        starting environment as well as evaluation success criteria. Only provided
-        fields will be updated.
+        """Update a Scenario.
+
+        Fields that are null will preserve the existing value. Fields
+        that are provided (including empty values) will replace the existing value
+        entirely.
 
         Args:
           environment_parameters: The Environment in which the Scenario will run.
 
           input_context: The input context for the Scenario.
 
-          metadata: User defined metadata to attach to the scenario for organization.
+          metadata: User defined metadata to attach to the scenario. Pass in empty map to clear.
 
-          name: Name of the scenario.
+          name: Name of the scenario. Cannot be blank.
 
           reference_output: A string representation of the reference output to solve the scenario. Commonly
               can be the result of a git diff or a sequence of command actions to apply to the
-              environment.
+              environment. Pass in empty string to clear.
 
-          required_environment_variables: Environment variables required to run the scenario.
+          required_environment_variables: Environment variables required to run the scenario. Pass in empty list to clear.
 
-          required_secret_names: Secrets required to run the scenario.
+          required_secret_names: Secrets required to run the scenario. Pass in empty list to clear.
 
           scoring_contract: The scoring contract for the Scenario.
 
-          validation_type: Validation strategy.
+          validation_type: Validation strategy. Pass in empty string to clear.
 
           extra_headers: Send extra headers
 
@@ -682,31 +683,32 @@ async def update(
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
         idempotency_key: str | None = None,
     ) -> ScenarioView:
-        """
-        Update a Scenario, a repeatable AI coding evaluation test that defines the
-        starting environment as well as evaluation success criteria. Only provided
-        fields will be updated.
+        """Update a Scenario.
+
+        Fields that are null will preserve the existing value. Fields
+        that are provided (including empty values) will replace the existing value
+        entirely.
 
         Args:
           environment_parameters: The Environment in which the Scenario will run.
 
           input_context: The input context for the Scenario.
 
-          metadata: User defined metadata to attach to the scenario for organization.
+          metadata: User defined metadata to attach to the scenario. Pass in empty map to clear.
 
-          name: Name of the scenario.
+          name: Name of the scenario. Cannot be blank.
 
           reference_output: A string representation of the reference output to solve the scenario. Commonly
               can be the result of a git diff or a sequence of command actions to apply to the
-              environment.
+              environment. Pass in empty string to clear.
 
-          required_environment_variables: Environment variables required to run the scenario.
+          required_environment_variables: Environment variables required to run the scenario. Pass in empty list to clear.
 
-          required_secret_names: Secrets required to run the scenario.
+          required_secret_names: Secrets required to run the scenario. Pass in empty list to clear.
 
           scoring_contract: The scoring contract for the Scenario.
 
-          validation_type: Validation strategy.
+          validation_type: Validation strategy. Pass in empty string to clear.
 
           extra_headers: Send extra headers
 

src/runloop_api_client/types/benchmark_create_params.py

@@ -12,7 +12,7 @@
 
 class BenchmarkCreateParams(TypedDict, total=False):
     name: Required[str]
-    """The name of the Benchmark. This must be unique."""
+    """The unique name of the Benchmark."""
 
     attribution: Optional[str]
     """Attribution information for the benchmark."""
@@ -21,12 +21,12 @@ class BenchmarkCreateParams(TypedDict, total=False):
     """Detailed description of the benchmark."""
 
     metadata: Optional[Dict[str, str]]
-    """User defined metadata to attach to the benchmark for organization."""
+    """User defined metadata to attach to the benchmark."""
 
     required_environment_variables: Optional[SequenceNotStr[str]]
     """Environment variables required to run the benchmark.
 
-    If any required variables are not supplied, the benchmark will fail to start
+    If any required variables are not supplied, the benchmark will fail to start.
     """
 
     required_secret_names: SequenceNotStr[str]

src/runloop_api_client/types/benchmark_update_params.py

@@ -3,38 +3,40 @@
 from __future__ import annotations
 
 from typing import Dict, Optional
-from typing_extensions import Required, TypedDict
+from typing_extensions import TypedDict
 
 from .._types import SequenceNotStr
 
 __all__ = ["BenchmarkUpdateParams"]
 
 
 class BenchmarkUpdateParams(TypedDict, total=False):
-    name: Required[str]
-    """The name of the Benchmark. This must be unique."""
-
     attribution: Optional[str]
-    """Attribution information for the benchmark."""
+    """Attribution information for the benchmark. Pass in empty string to clear."""
 
     description: Optional[str]
-    """Detailed description of the benchmark."""
+    """Detailed description of the benchmark. Pass in empty string to clear."""
 
     metadata: Optional[Dict[str, str]]
-    """User defined metadata to attach to the benchmark for organization."""
+    """User defined metadata to attach to the benchmark. Pass in empty map to clear."""
+
+    name: Optional[str]
+    """The unique name of the Benchmark. Cannot be blank."""
 
     required_environment_variables: Optional[SequenceNotStr[str]]
     """Environment variables required to run the benchmark.
 
-    If any required variables are not supplied, the benchmark will fail to start
+    If any required variables are not supplied, the benchmark will fail to start.
+    Pass in empty list to clear.
     """
 
-    required_secret_names: SequenceNotStr[str]
+    required_secret_names: Optional[SequenceNotStr[str]]
     """
     Secrets required to run the benchmark with (environment variable name will be
     mapped to the your user secret by name). If any of these secrets are not
-    provided or the mapping is incorrect, the benchmark will fail to start.
+    provided or the mapping is incorrect, the benchmark will fail to start. Pass in
+    empty list to clear.
     """
 
     scenario_ids: Optional[SequenceNotStr[str]]
-    """The Scenario IDs that make up the Benchmark."""
+    """The Scenario IDs that make up the Benchmark. Pass in empty list to clear."""

src/runloop_api_client/types/scenario_update_params.py

@@ -21,26 +21,29 @@ class ScenarioUpdateParams(TypedDict, total=False):
     """The input context for the Scenario."""
 
     metadata: Optional[Dict[str, str]]
-    """User defined metadata to attach to the scenario for organization."""
+    """User defined metadata to attach to the scenario. Pass in empty map to clear."""
 
     name: Optional[str]
-    """Name of the scenario."""
+    """Name of the scenario. Cannot be blank."""
 
     reference_output: Optional[str]
     """A string representation of the reference output to solve the scenario.
 
     Commonly can be the result of a git diff or a sequence of command actions to
-    apply to the environment.
+    apply to the environment. Pass in empty string to clear.
     """
 
     required_environment_variables: Optional[SequenceNotStr[str]]
-    """Environment variables required to run the scenario."""
+    """Environment variables required to run the scenario.
+
+    Pass in empty list to clear.
+    """
 
     required_secret_names: Optional[SequenceNotStr[str]]
-    """Secrets required to run the scenario."""
+    """Secrets required to run the scenario. Pass in empty list to clear."""
 
     scoring_contract: Optional[ScoringContractUpdateParam]
     """The scoring contract for the Scenario."""
 
     validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]]
-    """Validation strategy."""
+    """Validation strategy. Pass in empty string to clear."""