Skip to content

Commit 6615183

Browse files
stainless-app[bot]stainless-app[bot]
committed
feat(benchmarks): scenario scoring timeout (#7300)
1 parent 94e6c44 commit 6615183

6 files changed

Lines changed: 31 additions & 2 deletions

File tree

‎.stats.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
configured_endpoints: 117
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-15b7fd06098d8cb3639efb3b401a03d7d97a8ab4960b08077aa871f0cacb33d3.yml
3-
openapi_spec_hash: 93ab2fe88f9e57d8f262ad6d1179190e
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-ddfb27dde3b560eea5c53bfc8489439ace1501a7bb6a44aaeb3fd4859c6b65da.yml
3+
openapi_spec_hash: 47021a450cbaa18b225148c5db188dd4
44
config_hash: eb28692edd68a6ae95cf92af931c9976

‎src/runloop_api_client/resources/scenarios/scenarios.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ def create(
9494
reference_output: Optional[str] | Omit = omit,
9595
required_environment_variables: Optional[SequenceNotStr[str]] | Omit = omit,
9696
required_secret_names: Optional[SequenceNotStr[str]] | Omit = omit,
97+
scorer_timeout_sec: Optional[int] | Omit = omit,
9798
validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]] | Omit = omit,
9899
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
99100
# The extra values given here take precedence over values defined on the client or passed to this method.
@@ -129,6 +130,8 @@ def create(
129130
secret name). If these secrets are not provided or the mapping is incorrect, the
130131
scenario will fail to start.
131132
133+
scorer_timeout_sec: Timeout for scoring in seconds. Default 30 minutes (1800s).
134+
132135
validation_type: Validation strategy.
133136
134137
extra_headers: Send extra headers
@@ -153,6 +156,7 @@ def create(
153156
"reference_output": reference_output,
154157
"required_environment_variables": required_environment_variables,
155158
"required_secret_names": required_secret_names,
159+
"scorer_timeout_sec": scorer_timeout_sec,
156160
"validation_type": validation_type,
157161
},
158162
scenario_create_params.ScenarioCreateParams,
@@ -211,6 +215,7 @@ def update(
211215
reference_output: Optional[str] | Omit = omit,
212216
required_environment_variables: Optional[SequenceNotStr[str]] | Omit = omit,
213217
required_secret_names: Optional[SequenceNotStr[str]] | Omit = omit,
218+
scorer_timeout_sec: Optional[int] | Omit = omit,
214219
scoring_contract: Optional[ScoringContractUpdateParam] | Omit = omit,
215220
validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]] | Omit = omit,
216221
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
@@ -244,6 +249,8 @@ def update(
244249
245250
required_secret_names: Secrets required to run the scenario. Pass in empty list to clear.
246251
252+
scorer_timeout_sec: Timeout for scoring in seconds. Default 30 minutes (1800s).
253+
247254
scoring_contract: The scoring contract for the Scenario.
248255
249256
validation_type: Validation strategy. Pass in empty string to clear.
@@ -271,6 +278,7 @@ def update(
271278
"reference_output": reference_output,
272279
"required_environment_variables": required_environment_variables,
273280
"required_secret_names": required_secret_names,
281+
"scorer_timeout_sec": scorer_timeout_sec,
274282
"scoring_contract": scoring_contract,
275283
"validation_type": validation_type,
276284
},
@@ -556,6 +564,7 @@ async def create(
556564
reference_output: Optional[str] | Omit = omit,
557565
required_environment_variables: Optional[SequenceNotStr[str]] | Omit = omit,
558566
required_secret_names: Optional[SequenceNotStr[str]] | Omit = omit,
567+
scorer_timeout_sec: Optional[int] | Omit = omit,
559568
validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]] | Omit = omit,
560569
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
561570
# The extra values given here take precedence over values defined on the client or passed to this method.
@@ -591,6 +600,8 @@ async def create(
591600
secret name). If these secrets are not provided or the mapping is incorrect, the
592601
scenario will fail to start.
593602
603+
scorer_timeout_sec: Timeout for scoring in seconds. Default 30 minutes (1800s).
604+
594605
validation_type: Validation strategy.
595606
596607
extra_headers: Send extra headers
@@ -615,6 +626,7 @@ async def create(
615626
"reference_output": reference_output,
616627
"required_environment_variables": required_environment_variables,
617628
"required_secret_names": required_secret_names,
629+
"scorer_timeout_sec": scorer_timeout_sec,
618630
"validation_type": validation_type,
619631
},
620632
scenario_create_params.ScenarioCreateParams,
@@ -673,6 +685,7 @@ async def update(
673685
reference_output: Optional[str] | Omit = omit,
674686
required_environment_variables: Optional[SequenceNotStr[str]] | Omit = omit,
675687
required_secret_names: Optional[SequenceNotStr[str]] | Omit = omit,
688+
scorer_timeout_sec: Optional[int] | Omit = omit,
676689
scoring_contract: Optional[ScoringContractUpdateParam] | Omit = omit,
677690
validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]] | Omit = omit,
678691
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
@@ -706,6 +719,8 @@ async def update(
706719
707720
required_secret_names: Secrets required to run the scenario. Pass in empty list to clear.
708721
722+
scorer_timeout_sec: Timeout for scoring in seconds. Default 30 minutes (1800s).
723+
709724
scoring_contract: The scoring contract for the Scenario.
710725
711726
validation_type: Validation strategy. Pass in empty string to clear.
@@ -733,6 +748,7 @@ async def update(
733748
"reference_output": reference_output,
734749
"required_environment_variables": required_environment_variables,
735750
"required_secret_names": required_secret_names,
751+
"scorer_timeout_sec": scorer_timeout_sec,
736752
"scoring_contract": scoring_contract,
737753
"validation_type": validation_type,
738754
},

‎src/runloop_api_client/types/scenario_create_params.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,8 @@ class ScenarioCreateParams(TypedDict, total=False):
4949
scenario will fail to start.
5050
"""
5151

52+
scorer_timeout_sec: Optional[int]
53+
"""Timeout for scoring in seconds. Default 30 minutes (1800s)."""
54+
5255
validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]]
5356
"""Validation strategy."""

‎src/runloop_api_client/types/scenario_update_params.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ class ScenarioUpdateParams(TypedDict, total=False):
4242
required_secret_names: Optional[SequenceNotStr[str]]
4343
"""Secrets required to run the scenario. Pass in empty list to clear."""
4444

45+
scorer_timeout_sec: Optional[int]
46+
"""Timeout for scoring in seconds. Default 30 minutes (1800s)."""
47+
4548
scoring_contract: Optional[ScoringContractUpdateParam]
4649
"""The scoring contract for the Scenario."""
4750

‎src/runloop_api_client/types/scenario_view.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,5 +57,8 @@ class ScenarioView(BaseModel):
5757
If any required secrets are missing, the scenario will fail to start.
5858
"""
5959

60+
scorer_timeout_sec: Optional[int] = None
61+
"""Timeout for scoring in seconds. Default 30 minutes (1800s)."""
62+
6063
validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]] = None
6164
"""Validation strategy."""

‎tests/api_resources/test_scenarios.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ def test_method_create_with_all_params(self, client: Runloop) -> None:
9393
reference_output="reference_output",
9494
required_environment_variables=["string"],
9595
required_secret_names=["string"],
96+
scorer_timeout_sec=0,
9697
validation_type="UNSPECIFIED",
9798
)
9899
assert_matches_type(ScenarioView, scenario, path=["response"])
@@ -232,6 +233,7 @@ def test_method_update_with_all_params(self, client: Runloop) -> None:
232233
reference_output="reference_output",
233234
required_environment_variables=["string"],
234235
required_secret_names=["string"],
236+
scorer_timeout_sec=0,
235237
scoring_contract={
236238
"scoring_function_parameters": [
237239
{
@@ -502,6 +504,7 @@ async def test_method_create_with_all_params(self, async_client: AsyncRunloop) -
502504
reference_output="reference_output",
503505
required_environment_variables=["string"],
504506
required_secret_names=["string"],
507+
scorer_timeout_sec=0,
505508
validation_type="UNSPECIFIED",
506509
)
507510
assert_matches_type(ScenarioView, scenario, path=["response"])
@@ -641,6 +644,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncRunloop) -
641644
reference_output="reference_output",
642645
required_environment_variables=["string"],
643646
required_secret_names=["string"],
647+
scorer_timeout_sec=0,
644648
scoring_contract={
645649
"scoring_function_parameters": [
646650
{

0 commit comments

Comments
 (0)