Skip to content

Commit 33624ce

Browse files
stainless-app[bot]stainless-app[bot]
committed
refactor(benchmark): Deprecate /benchmark/{id}/runs in favor of /benchmark_runs (#7019)
1 parent 3b8cca8 commit 33624ce

9 files changed

Lines changed: 420 additions & 28 deletions

File tree

‎.stats.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
configured_endpoints: 103
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-5bbc26893c51a60089ff83a9d067c72252989fbfc5fa941cabf801f62b314c77.yml
3-
openapi_spec_hash: b5d826e352ca1ac815b7530278f4a5ab
4-
config_hash: ec63e62d4660c9c924b39a8671c2fdb1
1+
configured_endpoints: 108
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-e583f34fdcdc18499c8692e8eb8021f6163201f0f77206934c712c319a674d43.yml
3+
openapi_spec_hash: f2fb3f7f5c1f62d3dc397cd02cd1007a
4+
config_hash: 42959fa2708796cc2f83937278dde733

‎api.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,22 @@ Methods:
4545
- <code title="post /v1/benchmarks/runs/{id}/complete">client.benchmarks.runs.<a href="./src/resources/benchmarks/runs.ts">complete</a>(id) -> BenchmarkRunView</code>
4646
- <code title="get /v1/benchmarks/runs/{id}/scenario_runs">client.benchmarks.runs.<a href="./src/resources/benchmarks/runs.ts">listScenarioRuns</a>(id, { ...params }) -> ScenarioRunViewsBenchmarkRunsCursorIDPage</code>
4747

48+
# BenchmarkRuns
49+
50+
Types:
51+
52+
- <code><a href="./src/resources/benchmark-runs.ts">BenchmarkRunListView</a></code>
53+
- <code><a href="./src/resources/benchmark-runs.ts">BenchmarkRunView</a></code>
54+
- <code><a href="./src/resources/benchmark-runs.ts">ScenarioRunListView</a></code>
55+
56+
Methods:
57+
58+
- <code title="get /v1/benchmark_runs/{id}">client.benchmarkRuns.<a href="./src/resources/benchmark-runs.ts">retrieve</a>(id) -> BenchmarkRunView</code>
59+
- <code title="get /v1/benchmark_runs">client.benchmarkRuns.<a href="./src/resources/benchmark-runs.ts">list</a>({ ...params }) -> BenchmarkRunViewsBenchmarkRunsCursorIDPage</code>
60+
- <code title="post /v1/benchmark_runs/{id}/cancel">client.benchmarkRuns.<a href="./src/resources/benchmark-runs.ts">cancel</a>(id) -> BenchmarkRunView</code>
61+
- <code title="post /v1/benchmark_runs/{id}/complete">client.benchmarkRuns.<a href="./src/resources/benchmark-runs.ts">complete</a>(id) -> BenchmarkRunView</code>
62+
- <code title="get /v1/benchmark_runs/{id}/scenario_runs">client.benchmarkRuns.<a href="./src/resources/benchmark-runs.ts">listScenarioRuns</a>(id, { ...params }) -> ScenarioRunViewsBenchmarkRunsCursorIDPage</code>
63+
4864
# Agents
4965

5066
Types:

‎src/index.ts‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,15 @@ import {
4141
AgentViewsAgentsCursorIDPage,
4242
Agents,
4343
} from './resources/agents';
44+
import {
45+
BenchmarkRunListParams,
46+
BenchmarkRunListScenarioRunsParams,
47+
BenchmarkRunListView,
48+
BenchmarkRunView,
49+
BenchmarkRunViewsBenchmarkRunsCursorIDPage,
50+
BenchmarkRuns,
51+
ScenarioRunListView,
52+
} from './resources/benchmark-runs';
4453
import {
4554
BlueprintBuildFromInspectionParameters,
4655
BlueprintBuildLog,
@@ -118,8 +127,8 @@ import {
118127
BenchmarkDefinitionsParams,
119128
BenchmarkListParams,
120129
BenchmarkListPublicParams,
121-
BenchmarkRunListView,
122-
BenchmarkRunView,
130+
BenchmarkRunListView as BenchmarksAPIBenchmarkRunListView,
131+
BenchmarkRunView as BenchmarksAPIBenchmarkRunView,
123132
BenchmarkScenarioUpdateParameters,
124133
BenchmarkStartRunParams,
125134
BenchmarkUpdateParameters,
@@ -176,7 +185,7 @@ import {
176185
ScenarioEnvironment,
177186
ScenarioListParams,
178187
ScenarioListPublicParams,
179-
ScenarioRunListView,
188+
ScenarioRunListView as ScenariosAPIScenarioRunListView,
180189
ScenarioRunView,
181190
ScenarioStartRunParams,
182191
ScenarioUpdateParameters,
@@ -317,6 +326,7 @@ export class Runloop extends Core.APIClient {
317326
}
318327

319328
benchmarks: API.Benchmarks = new API.Benchmarks(this);
329+
benchmarkRuns: API.BenchmarkRuns = new API.BenchmarkRuns(this);
320330
agents: API.Agents = new API.Agents(this);
321331
blueprints: API.Blueprints = new API.Blueprints(this);
322332
devboxes: API.Devboxes = new API.Devboxes(this);
@@ -371,6 +381,8 @@ export class Runloop extends Core.APIClient {
371381

372382
Runloop.Benchmarks = Benchmarks;
373383
Runloop.BenchmarkViewsBenchmarksCursorIDPage = BenchmarkViewsBenchmarksCursorIDPage;
384+
Runloop.BenchmarkRuns = BenchmarkRuns;
385+
Runloop.BenchmarkRunViewsBenchmarkRunsCursorIDPage = BenchmarkRunViewsBenchmarkRunsCursorIDPage;
374386
Runloop.Agents = Agents;
375387
Runloop.AgentViewsAgentsCursorIDPage = AgentViewsAgentsCursorIDPage;
376388
Runloop.Blueprints = Blueprints;
@@ -466,8 +478,8 @@ export declare namespace Runloop {
466478
export {
467479
Benchmarks as Benchmarks,
468480
type BenchmarkCreateParameters as BenchmarkCreateParameters,
469-
type BenchmarkRunListView as BenchmarkRunListView,
470-
type BenchmarkRunView as BenchmarkRunView,
481+
type BenchmarksAPIBenchmarkRunListView as BenchmarkRunListView,
482+
type BenchmarksAPIBenchmarkRunView as BenchmarkRunView,
471483
type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters,
472484
type BenchmarkUpdateParameters as BenchmarkUpdateParameters,
473485
type BenchmarkView as BenchmarkView,
@@ -483,6 +495,16 @@ export declare namespace Runloop {
483495
type BenchmarkUpdateScenariosParams as BenchmarkUpdateScenariosParams,
484496
};
485497

498+
export {
499+
BenchmarkRuns as BenchmarkRuns,
500+
type BenchmarkRunListView as BenchmarkRunListView,
501+
type BenchmarkRunView as BenchmarkRunView,
502+
type ScenarioRunListView as ScenarioRunListView,
503+
BenchmarkRunViewsBenchmarkRunsCursorIDPage as BenchmarkRunViewsBenchmarkRunsCursorIDPage,
504+
type BenchmarkRunListParams as BenchmarkRunListParams,
505+
type BenchmarkRunListScenarioRunsParams as BenchmarkRunListScenarioRunsParams,
506+
};
507+
486508
export {
487509
Agents as Agents,
488510
type AgentCreateParameters as AgentCreateParameters,
@@ -556,7 +578,7 @@ export declare namespace Runloop {
556578
type InputContextUpdate as InputContextUpdate,
557579
type ScenarioCreateParameters as ScenarioCreateParameters,
558580
type ScenarioEnvironment as ScenarioEnvironment,
559-
type ScenarioRunListView as ScenarioRunListView,
581+
type ScenariosAPIScenarioRunListView as ScenarioRunListView,
560582
type ScenarioRunView as ScenarioRunView,
561583
type ScenarioUpdateParameters as ScenarioUpdateParameters,
562584
type ScenarioView as ScenarioView,

‎src/resources/benchmark-runs.ts‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2+
3+
import { APIResource } from '../resource';
4+
import { isRequestOptions } from '../core';
5+
import * as Core from '../core';
6+
import * as ScenariosAPI from './scenarios/scenarios';
7+
import { ScenarioRunViewsBenchmarkRunsCursorIDPage } from './scenarios/scenarios';
8+
import { BenchmarkRunsCursorIDPage, type BenchmarkRunsCursorIDPageParams } from '../pagination';
9+
10+
export class BenchmarkRuns extends APIResource {
11+
/**
12+
* Get a BenchmarkRun given ID.
13+
*/
14+
retrieve(id: string, options?: Core.RequestOptions): Core.APIPromise<BenchmarkRunView> {
15+
return this._client.get(`/v1/benchmark_runs/${id}`, options);
16+
}
17+
18+
/**
19+
* List all BenchmarkRuns matching filter.
20+
*/
21+
list(
22+
query?: BenchmarkRunListParams,
23+
options?: Core.RequestOptions,
24+
): Core.PagePromise<BenchmarkRunViewsBenchmarkRunsCursorIDPage, BenchmarkRunView>;
25+
list(
26+
options?: Core.RequestOptions,
27+
): Core.PagePromise<BenchmarkRunViewsBenchmarkRunsCursorIDPage, BenchmarkRunView>;
28+
list(
29+
query: BenchmarkRunListParams | Core.RequestOptions = {},
30+
options?: Core.RequestOptions,
31+
): Core.PagePromise<BenchmarkRunViewsBenchmarkRunsCursorIDPage, BenchmarkRunView> {
32+
if (isRequestOptions(query)) {
33+
return this.list({}, query);
34+
}
35+
return this._client.getAPIList('/v1/benchmark_runs', BenchmarkRunViewsBenchmarkRunsCursorIDPage, {
36+
query,
37+
...options,
38+
});
39+
}
40+
41+
/**
42+
* Cancel a currently running Benchmark run.
43+
*/
44+
cancel(id: string, options?: Core.RequestOptions): Core.APIPromise<BenchmarkRunView> {
45+
return this._client.post(`/v1/benchmark_runs/${id}/cancel`, options);
46+
}
47+
48+
/**
49+
* Complete a currently running BenchmarkRun.
50+
*/
51+
complete(id: string, options?: Core.RequestOptions): Core.APIPromise<BenchmarkRunView> {
52+
return this._client.post(`/v1/benchmark_runs/${id}/complete`, options);
53+
}
54+
55+
/**
56+
* List started scenario runs for a benchmark run.
57+
*/
58+
listScenarioRuns(
59+
id: string,
60+
query?: BenchmarkRunListScenarioRunsParams,
61+
options?: Core.RequestOptions,
62+
): Core.PagePromise<ScenarioRunViewsBenchmarkRunsCursorIDPage, ScenariosAPI.ScenarioRunView>;
63+
listScenarioRuns(
64+
id: string,
65+
options?: Core.RequestOptions,
66+
): Core.PagePromise<ScenarioRunViewsBenchmarkRunsCursorIDPage, ScenariosAPI.ScenarioRunView>;
67+
listScenarioRuns(
68+
id: string,
69+
query: BenchmarkRunListScenarioRunsParams | Core.RequestOptions = {},
70+
options?: Core.RequestOptions,
71+
): Core.PagePromise<ScenarioRunViewsBenchmarkRunsCursorIDPage, ScenariosAPI.ScenarioRunView> {
72+
if (isRequestOptions(query)) {
73+
return this.listScenarioRuns(id, {}, query);
74+
}
75+
return this._client.getAPIList(
76+
`/v1/benchmark_runs/${id}/scenario_runs`,
77+
ScenarioRunViewsBenchmarkRunsCursorIDPage,
78+
{ query, ...options },
79+
);
80+
}
81+
}
82+
83+
export class BenchmarkRunViewsBenchmarkRunsCursorIDPage extends BenchmarkRunsCursorIDPage<BenchmarkRunView> {}
84+
85+
export interface BenchmarkRunListView {
86+
has_more: boolean;
87+
88+
remaining_count: number;
89+
90+
/**
91+
* List of BenchmarkRuns matching filter.
92+
*/
93+
runs: Array<BenchmarkRunView>;
94+
95+
total_count: number;
96+
}
97+
98+
/**
99+
* A BenchmarkRunView represents a run of a complete set of Scenarios, organized
100+
* under a Benchmark or created by a BenchmarkJob.
101+
*/
102+
export interface BenchmarkRunView {
103+
/**
104+
* The ID of the BenchmarkRun.
105+
*/
106+
id: string;
107+
108+
/**
109+
* User defined metadata to attach to the benchmark run for organization.
110+
*/
111+
metadata: { [key: string]: string };
112+
113+
/**
114+
* The time the benchmark run execution started (Unix timestamp milliseconds).
115+
*/
116+
start_time_ms: number;
117+
118+
/**
119+
* The state of the BenchmarkRun.
120+
*/
121+
state: 'running' | 'canceled' | 'completed';
122+
123+
/**
124+
* The ID of the Benchmark definition. Present if run was created from a benchmark
125+
* definition.
126+
*/
127+
benchmark_id?: string | null;
128+
129+
/**
130+
* The duration for the BenchmarkRun to complete.
131+
*/
132+
duration_ms?: number | null;
133+
134+
/**
135+
* Environment variables used to run the benchmark.
136+
*/
137+
environment_variables?: { [key: string]: string } | null;
138+
139+
/**
140+
* The name of the BenchmarkRun.
141+
*/
142+
name?: string | null;
143+
144+
/**
145+
* Purpose of the run.
146+
*/
147+
purpose?: string | null;
148+
149+
/**
150+
* The final score across the BenchmarkRun, present once completed. Calculated as
151+
* sum of scenario scores / number of scenario runs.
152+
*/
153+
score?: number | null;
154+
155+
/**
156+
* User secrets used to run the benchmark. Example: {"DB_PASS":
157+
* "DATABASE_PASSWORD"} would set the environment variable 'DB_PASS' on all
158+
* scenario devboxes to the value of the secret 'DATABASE_PASSWORD'.
159+
*/
160+
secrets_provided?: { [key: string]: string } | null;
161+
}
162+
163+
export interface ScenarioRunListView {
164+
has_more: boolean;
165+
166+
remaining_count: number;
167+
168+
/**
169+
* List of ScenarioRuns matching filter.
170+
*/
171+
runs: Array<ScenariosAPI.ScenarioRunView>;
172+
173+
total_count: number;
174+
}
175+
176+
export interface BenchmarkRunListParams extends BenchmarkRunsCursorIDPageParams {
177+
/**
178+
* The Benchmark ID to filter by.
179+
*/
180+
benchmark_id?: string;
181+
182+
/**
183+
* Filter by name
184+
*/
185+
name?: string;
186+
}
187+
188+
export interface BenchmarkRunListScenarioRunsParams extends BenchmarkRunsCursorIDPageParams {
189+
/**
190+
* Filter by Scenario Run state
191+
*/
192+
state?: 'running' | 'scoring' | 'scored' | 'completed' | 'canceled' | 'timeout' | 'failed';
193+
}
194+
195+
BenchmarkRuns.BenchmarkRunViewsBenchmarkRunsCursorIDPage = BenchmarkRunViewsBenchmarkRunsCursorIDPage;
196+
197+
export declare namespace BenchmarkRuns {
198+
export {
199+
type BenchmarkRunListView as BenchmarkRunListView,
200+
type BenchmarkRunView as BenchmarkRunView,
201+
type ScenarioRunListView as ScenarioRunListView,
202+
BenchmarkRunViewsBenchmarkRunsCursorIDPage as BenchmarkRunViewsBenchmarkRunsCursorIDPage,
203+
type BenchmarkRunListParams as BenchmarkRunListParams,
204+
type BenchmarkRunListScenarioRunsParams as BenchmarkRunListScenarioRunsParams,
205+
};
206+
}
207+
208+
export { ScenarioRunViewsBenchmarkRunsCursorIDPage };

‎src/resources/benchmarks/benchmarks.ts‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,12 @@
33
import { APIResource } from '../../resource';
44
import { isRequestOptions } from '../../core';
55
import * as Core from '../../core';
6+
import * as BenchmarkRunsAPI from '../benchmark-runs';
67
import * as Shared from '../shared';
78
import * as RunsAPI from './runs';
89
import { RunListParams, RunListScenarioRunsParams, Runs } from './runs';
910
import * as ScenariosAPI from '../scenarios/scenarios';
10-
import {
11-
BenchmarkRunsCursorIDPage,
12-
BenchmarksCursorIDPage,
13-
type BenchmarksCursorIDPageParams,
14-
} from '../../pagination';
11+
import { BenchmarksCursorIDPage, type BenchmarksCursorIDPageParams } from '../../pagination';
1512

1613
export class Benchmarks extends APIResource {
1714
runs: RunsAPI.Runs = new RunsAPI.Runs(this._client);
@@ -119,7 +116,10 @@ export class Benchmarks extends APIResource {
119116
/**
120117
* Start a new BenchmarkRun based on the provided Benchmark.
121118
*/
122-
startRun(body: BenchmarkStartRunParams, options?: Core.RequestOptions): Core.APIPromise<BenchmarkRunView> {
119+
startRun(
120+
body: BenchmarkStartRunParams,
121+
options?: Core.RequestOptions,
122+
): Core.APIPromise<BenchmarkRunsAPI.BenchmarkRunView> {
123123
return this._client.post('/v1/benchmarks/start_run', { body, ...options });
124124
}
125125

@@ -146,8 +146,6 @@ export class Benchmarks extends APIResource {
146146

147147
export class BenchmarkViewsBenchmarksCursorIDPage extends BenchmarksCursorIDPage<BenchmarkView> {}
148148

149-
export class BenchmarkRunViewsBenchmarkRunsCursorIDPage extends BenchmarkRunsCursorIDPage<BenchmarkRunView> {}
150-
151149
/**
152150
* BenchmarkCreateParameters contain the set of parameters to create a Benchmark.
153151
*/
@@ -199,7 +197,7 @@ export interface BenchmarkRunListView {
199197
/**
200198
* List of BenchmarkRuns matching filter.
201199
*/
202-
runs: Array<BenchmarkRunView>;
200+
runs: Array<BenchmarkRunsAPI.BenchmarkRunView>;
203201

204202
total_count: number;
205203
}

‎src/resources/benchmarks/index.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
22

33
export {
4-
BenchmarkRunViewsBenchmarkRunsCursorIDPage,
54
BenchmarkViewsBenchmarksCursorIDPage,
65
Benchmarks,
76
type BenchmarkCreateParameters,

0 commit comments

Comments
 (0)