From 038fc8a393d533d4a29532b570fc7b48ef3292f3 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 23:40:08 +0000 Subject: [PATCH 1/7] feat(blueprint): Set cilium network policy on blueprint build (#7006) --- .stats.yml | 4 +- src/resources/blueprints.ts | 39 +++++++++++++++++++ src/resources/devboxes/devboxes.ts | 11 +++--- tests/api-resources/blueprints.test.ts | 3 ++ tests/api-resources/devboxes/devboxes.test.ts | 1 - 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/.stats.yml b/.stats.yml index de2b8be94..a3fb034e6 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 103 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-8ebd9dbfacb180ec9c980f5e46a814b0c8a3cb6063136db18780271d33a8dfa6.yml -openapi_spec_hash: 12d4edec72b722945bade9419636d63e +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-eac4defbf39825fa88e0be11955be80cd004bcea46398be52f79800570539be0.yml +openapi_spec_hash: d18585a489895642807922dfa558226d config_hash: ec63e62d4660c9c924b39a8671c2fdb1 diff --git a/src/resources/blueprints.ts b/src/resources/blueprints.ts index 37eb2717c..650a36130 100644 --- a/src/resources/blueprints.ts +++ b/src/resources/blueprints.ts @@ -251,6 +251,12 @@ export interface BlueprintBuildFromInspectionParameters { */ metadata?: { [key: string]: string } | null; + /** + * (Optional) ID of the network policy to apply during blueprint build. This + * restricts network access during the build process. + */ + network_policy_id?: string | null; + /** * (Optional) Map of mount IDs/environment variable names to secret names. Secrets * can be used as environment variables in system_setup_commands. Example: @@ -348,6 +354,15 @@ export interface BlueprintBuildParameters { */ metadata?: { [key: string]: string } | null; + /** + * (Optional) ID of the network policy to apply during blueprint build. This + * restricts network access during the build process. This does not affect devboxes + * created from this blueprint; if you want devboxes created from this blueprint to + * inherit the network policy, set the network_policy_id on the blueprint launch + * parameters. + */ + network_policy_id?: string | null; + /** * (Optional) Map of mount IDs/environment variable names to secret names. Secrets * will be available to commands during the build. Secrets are NOT stored in the @@ -657,6 +672,15 @@ export interface BlueprintCreateParams { */ metadata?: { [key: string]: string } | null; + /** + * (Optional) ID of the network policy to apply during blueprint build. This + * restricts network access during the build process. This does not affect devboxes + * created from this blueprint; if you want devboxes created from this blueprint to + * inherit the network policy, set the network_policy_id on the blueprint launch + * parameters. + */ + network_policy_id?: string | null; + /** * (Optional) Map of mount IDs/environment variable names to secret names. Secrets * will be available to commands during the build. Secrets are NOT stored in the @@ -782,6 +806,12 @@ export interface BlueprintCreateFromInspectionParams { */ metadata?: { [key: string]: string } | null; + /** + * (Optional) ID of the network policy to apply during blueprint build. This + * restricts network access during the build process. + */ + network_policy_id?: string | null; + /** * (Optional) Map of mount IDs/environment variable names to secret names. Secrets * can be used as environment variables in system_setup_commands. Example: @@ -862,6 +892,15 @@ export interface BlueprintPreviewParams { */ metadata?: { [key: string]: string } | null; + /** + * (Optional) ID of the network policy to apply during blueprint build. This + * restricts network access during the build process. This does not affect devboxes + * created from this blueprint; if you want devboxes created from this blueprint to + * inherit the network policy, set the network_policy_id on the blueprint launch + * parameters. + */ + network_policy_id?: string | null; + /** * (Optional) Map of mount IDs/environment variable names to secret names. Secrets * will be available to commands during the build. Secrets are NOT stored in the diff --git a/src/resources/devboxes/devboxes.ts b/src/resources/devboxes/devboxes.ts index ad482b450..a5f4ac923 100644 --- a/src/resources/devboxes/devboxes.ts +++ b/src/resources/devboxes/devboxes.ts @@ -821,6 +821,11 @@ export interface DevboxView { */ name?: string | null; + /** + * The network policy ID used to create the Devbox. + */ + network_policy_id?: string | null; + /** * The shutdown reason if the Devbox shutdown, if the Devbox has a 'shutdown' * status. @@ -960,12 +965,6 @@ export interface DevboxCreateParams { */ name?: string | null; - /** - * (Optional) ID of the network policy to apply to this Devbox. If not specified, - * the default network policy will be used. - */ - network_policy_id?: string | null; - /** * Repository connection id the devbox should source its base image from. */ diff --git a/tests/api-resources/blueprints.test.ts b/tests/api-resources/blueprints.test.ts index 588a0b772..888ae59fa 100644 --- a/tests/api-resources/blueprints.test.ts +++ b/tests/api-resources/blueprints.test.ts @@ -52,6 +52,7 @@ describe('resource blueprints', () => { user_parameters: { uid: 0, username: 'username' }, }, metadata: { foo: 'string' }, + network_policy_id: 'network_policy_id', secrets: { foo: 'string' }, services: [ { @@ -195,6 +196,7 @@ describe('resource blueprints', () => { user_parameters: { uid: 0, username: 'username' }, }, metadata: { foo: 'string' }, + network_policy_id: 'network_policy_id', secrets: { foo: 'string' }, system_setup_commands: ['string'], }); @@ -293,6 +295,7 @@ describe('resource blueprints', () => { user_parameters: { uid: 0, username: 'username' }, }, metadata: { foo: 'string' }, + network_policy_id: 'network_policy_id', secrets: { foo: 'string' }, services: [ { diff --git a/tests/api-resources/devboxes/devboxes.test.ts b/tests/api-resources/devboxes/devboxes.test.ts index 0bd6c720b..87b88fdb4 100644 --- a/tests/api-resources/devboxes/devboxes.test.ts +++ b/tests/api-resources/devboxes/devboxes.test.ts @@ -69,7 +69,6 @@ describe('resource devboxes', () => { }, ], name: 'name', - network_policy_id: 'network_policy_id', repo_connection_id: 'repo_connection_id', secrets: { foo: 'string' }, snapshot_id: 'snapshot_id', From 849bcfab616e79e5b52612afd09ee3eab36d3f91 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 20 Jan 2026 00:05:49 +0000 Subject: [PATCH 2/7] chore(devbox): Remove network policy from devbox view; use launch params instead (#7025) --- .stats.yml | 4 ++-- src/resources/devboxes/devboxes.ts | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.stats.yml b/.stats.yml index a3fb034e6..6f6409f94 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 103 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-eac4defbf39825fa88e0be11955be80cd004bcea46398be52f79800570539be0.yml -openapi_spec_hash: d18585a489895642807922dfa558226d +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-5bbc26893c51a60089ff83a9d067c72252989fbfc5fa941cabf801f62b314c77.yml +openapi_spec_hash: b5d826e352ca1ac815b7530278f4a5ab config_hash: ec63e62d4660c9c924b39a8671c2fdb1 diff --git a/src/resources/devboxes/devboxes.ts b/src/resources/devboxes/devboxes.ts index a5f4ac923..55e604f2b 100644 --- a/src/resources/devboxes/devboxes.ts +++ b/src/resources/devboxes/devboxes.ts @@ -821,11 +821,6 @@ export interface DevboxView { */ name?: string | null; - /** - * The network policy ID used to create the Devbox. - */ - network_policy_id?: string | null; - /** * The shutdown reason if the Devbox shutdown, if the Devbox has a 'shutdown' * status. From 33624ce149003dfb1eaa49c10c8efa233b875ab9 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 20 Jan 2026 14:30:58 +0000 Subject: [PATCH 3/7] refactor(benchmark): Deprecate /benchmark/{id}/runs in favor of /benchmark_runs (#7019) --- .stats.yml | 8 +- api.md | 16 ++ src/index.ts | 34 +++- src/resources/benchmark-runs.ts | 208 +++++++++++++++++++++ src/resources/benchmarks/benchmarks.ts | 16 +- src/resources/benchmarks/index.ts | 1 - src/resources/benchmarks/runs.ts | 26 ++- src/resources/index.ts | 8 + tests/api-resources/benchmark-runs.test.ts | 131 +++++++++++++ 9 files changed, 420 insertions(+), 28 deletions(-) create mode 100644 src/resources/benchmark-runs.ts create mode 100644 tests/api-resources/benchmark-runs.test.ts diff --git a/.stats.yml b/.stats.yml index 6f6409f94..553c2ca6f 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 103 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-5bbc26893c51a60089ff83a9d067c72252989fbfc5fa941cabf801f62b314c77.yml -openapi_spec_hash: b5d826e352ca1ac815b7530278f4a5ab -config_hash: ec63e62d4660c9c924b39a8671c2fdb1 +configured_endpoints: 108 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-e583f34fdcdc18499c8692e8eb8021f6163201f0f77206934c712c319a674d43.yml +openapi_spec_hash: f2fb3f7f5c1f62d3dc397cd02cd1007a +config_hash: 42959fa2708796cc2f83937278dde733 diff --git a/api.md b/api.md index 544ce44ac..83b439780 100644 --- a/api.md +++ b/api.md @@ -45,6 +45,22 @@ Methods: - client.benchmarks.runs.complete(id) -> BenchmarkRunView - client.benchmarks.runs.listScenarioRuns(id, { ...params }) -> ScenarioRunViewsBenchmarkRunsCursorIDPage +# BenchmarkRuns + +Types: + +- BenchmarkRunListView +- BenchmarkRunView +- ScenarioRunListView + +Methods: + +- client.benchmarkRuns.retrieve(id) -> BenchmarkRunView +- client.benchmarkRuns.list({ ...params }) -> BenchmarkRunViewsBenchmarkRunsCursorIDPage +- client.benchmarkRuns.cancel(id) -> BenchmarkRunView +- client.benchmarkRuns.complete(id) -> BenchmarkRunView +- client.benchmarkRuns.listScenarioRuns(id, { ...params }) -> ScenarioRunViewsBenchmarkRunsCursorIDPage + # Agents Types: diff --git a/src/index.ts b/src/index.ts index dc2cb236d..bbce762fd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -41,6 +41,15 @@ import { AgentViewsAgentsCursorIDPage, Agents, } from './resources/agents'; +import { + BenchmarkRunListParams, + BenchmarkRunListScenarioRunsParams, + BenchmarkRunListView, + BenchmarkRunView, + BenchmarkRunViewsBenchmarkRunsCursorIDPage, + BenchmarkRuns, + ScenarioRunListView, +} from './resources/benchmark-runs'; import { BlueprintBuildFromInspectionParameters, BlueprintBuildLog, @@ -118,8 +127,8 @@ import { BenchmarkDefinitionsParams, BenchmarkListParams, BenchmarkListPublicParams, - BenchmarkRunListView, - BenchmarkRunView, + BenchmarkRunListView as BenchmarksAPIBenchmarkRunListView, + BenchmarkRunView as BenchmarksAPIBenchmarkRunView, BenchmarkScenarioUpdateParameters, BenchmarkStartRunParams, BenchmarkUpdateParameters, @@ -176,7 +185,7 @@ import { ScenarioEnvironment, ScenarioListParams, ScenarioListPublicParams, - ScenarioRunListView, + ScenarioRunListView as ScenariosAPIScenarioRunListView, ScenarioRunView, ScenarioStartRunParams, ScenarioUpdateParameters, @@ -317,6 +326,7 @@ export class Runloop extends Core.APIClient { } benchmarks: API.Benchmarks = new API.Benchmarks(this); + benchmarkRuns: API.BenchmarkRuns = new API.BenchmarkRuns(this); agents: API.Agents = new API.Agents(this); blueprints: API.Blueprints = new API.Blueprints(this); devboxes: API.Devboxes = new API.Devboxes(this); @@ -371,6 +381,8 @@ export class Runloop extends Core.APIClient { Runloop.Benchmarks = Benchmarks; Runloop.BenchmarkViewsBenchmarksCursorIDPage = BenchmarkViewsBenchmarksCursorIDPage; +Runloop.BenchmarkRuns = BenchmarkRuns; +Runloop.BenchmarkRunViewsBenchmarkRunsCursorIDPage = BenchmarkRunViewsBenchmarkRunsCursorIDPage; Runloop.Agents = Agents; Runloop.AgentViewsAgentsCursorIDPage = AgentViewsAgentsCursorIDPage; Runloop.Blueprints = Blueprints; @@ -466,8 +478,8 @@ export declare namespace Runloop { export { Benchmarks as Benchmarks, type BenchmarkCreateParameters as BenchmarkCreateParameters, - type BenchmarkRunListView as BenchmarkRunListView, - type BenchmarkRunView as BenchmarkRunView, + type BenchmarksAPIBenchmarkRunListView as BenchmarkRunListView, + type BenchmarksAPIBenchmarkRunView as BenchmarkRunView, type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters, type BenchmarkUpdateParameters as BenchmarkUpdateParameters, type BenchmarkView as BenchmarkView, @@ -483,6 +495,16 @@ export declare namespace Runloop { type BenchmarkUpdateScenariosParams as BenchmarkUpdateScenariosParams, }; + export { + BenchmarkRuns as BenchmarkRuns, + type BenchmarkRunListView as BenchmarkRunListView, + type BenchmarkRunView as BenchmarkRunView, + type ScenarioRunListView as ScenarioRunListView, + BenchmarkRunViewsBenchmarkRunsCursorIDPage as BenchmarkRunViewsBenchmarkRunsCursorIDPage, + type BenchmarkRunListParams as BenchmarkRunListParams, + type BenchmarkRunListScenarioRunsParams as BenchmarkRunListScenarioRunsParams, + }; + export { Agents as Agents, type AgentCreateParameters as AgentCreateParameters, @@ -556,7 +578,7 @@ export declare namespace Runloop { type InputContextUpdate as InputContextUpdate, type ScenarioCreateParameters as ScenarioCreateParameters, type ScenarioEnvironment as ScenarioEnvironment, - type ScenarioRunListView as ScenarioRunListView, + type ScenariosAPIScenarioRunListView as ScenarioRunListView, type ScenarioRunView as ScenarioRunView, type ScenarioUpdateParameters as ScenarioUpdateParameters, type ScenarioView as ScenarioView, diff --git a/src/resources/benchmark-runs.ts b/src/resources/benchmark-runs.ts new file mode 100644 index 000000000..1e83138be --- /dev/null +++ b/src/resources/benchmark-runs.ts @@ -0,0 +1,208 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import { APIResource } from '../resource'; +import { isRequestOptions } from '../core'; +import * as Core from '../core'; +import * as ScenariosAPI from './scenarios/scenarios'; +import { ScenarioRunViewsBenchmarkRunsCursorIDPage } from './scenarios/scenarios'; +import { BenchmarkRunsCursorIDPage, type BenchmarkRunsCursorIDPageParams } from '../pagination'; + +export class BenchmarkRuns extends APIResource { + /** + * Get a BenchmarkRun given ID. + */ + retrieve(id: string, options?: Core.RequestOptions): Core.APIPromise { + return this._client.get(`/v1/benchmark_runs/${id}`, options); + } + + /** + * List all BenchmarkRuns matching filter. + */ + list( + query?: BenchmarkRunListParams, + options?: Core.RequestOptions, + ): Core.PagePromise; + list( + options?: Core.RequestOptions, + ): Core.PagePromise; + list( + query: BenchmarkRunListParams | Core.RequestOptions = {}, + options?: Core.RequestOptions, + ): Core.PagePromise { + if (isRequestOptions(query)) { + return this.list({}, query); + } + return this._client.getAPIList('/v1/benchmark_runs', BenchmarkRunViewsBenchmarkRunsCursorIDPage, { + query, + ...options, + }); + } + + /** + * Cancel a currently running Benchmark run. + */ + cancel(id: string, options?: Core.RequestOptions): Core.APIPromise { + return this._client.post(`/v1/benchmark_runs/${id}/cancel`, options); + } + + /** + * Complete a currently running BenchmarkRun. + */ + complete(id: string, options?: Core.RequestOptions): Core.APIPromise { + return this._client.post(`/v1/benchmark_runs/${id}/complete`, options); + } + + /** + * List started scenario runs for a benchmark run. + */ + listScenarioRuns( + id: string, + query?: BenchmarkRunListScenarioRunsParams, + options?: Core.RequestOptions, + ): Core.PagePromise; + listScenarioRuns( + id: string, + options?: Core.RequestOptions, + ): Core.PagePromise; + listScenarioRuns( + id: string, + query: BenchmarkRunListScenarioRunsParams | Core.RequestOptions = {}, + options?: Core.RequestOptions, + ): Core.PagePromise { + if (isRequestOptions(query)) { + return this.listScenarioRuns(id, {}, query); + } + return this._client.getAPIList( + `/v1/benchmark_runs/${id}/scenario_runs`, + ScenarioRunViewsBenchmarkRunsCursorIDPage, + { query, ...options }, + ); + } +} + +export class BenchmarkRunViewsBenchmarkRunsCursorIDPage extends BenchmarkRunsCursorIDPage {} + +export interface BenchmarkRunListView { + has_more: boolean; + + remaining_count: number; + + /** + * List of BenchmarkRuns matching filter. + */ + runs: Array; + + total_count: number; +} + +/** + * A BenchmarkRunView represents a run of a complete set of Scenarios, organized + * under a Benchmark or created by a BenchmarkJob. + */ +export interface BenchmarkRunView { + /** + * The ID of the BenchmarkRun. + */ + id: string; + + /** + * User defined metadata to attach to the benchmark run for organization. + */ + metadata: { [key: string]: string }; + + /** + * The time the benchmark run execution started (Unix timestamp milliseconds). + */ + start_time_ms: number; + + /** + * The state of the BenchmarkRun. + */ + state: 'running' | 'canceled' | 'completed'; + + /** + * The ID of the Benchmark definition. Present if run was created from a benchmark + * definition. + */ + benchmark_id?: string | null; + + /** + * The duration for the BenchmarkRun to complete. + */ + duration_ms?: number | null; + + /** + * Environment variables used to run the benchmark. + */ + environment_variables?: { [key: string]: string } | null; + + /** + * The name of the BenchmarkRun. + */ + name?: string | null; + + /** + * Purpose of the run. + */ + purpose?: string | null; + + /** + * The final score across the BenchmarkRun, present once completed. Calculated as + * sum of scenario scores / number of scenario runs. + */ + score?: number | null; + + /** + * User secrets used to run the benchmark. Example: {"DB_PASS": + * "DATABASE_PASSWORD"} would set the environment variable 'DB_PASS' on all + * scenario devboxes to the value of the secret 'DATABASE_PASSWORD'. + */ + secrets_provided?: { [key: string]: string } | null; +} + +export interface ScenarioRunListView { + has_more: boolean; + + remaining_count: number; + + /** + * List of ScenarioRuns matching filter. + */ + runs: Array; + + total_count: number; +} + +export interface BenchmarkRunListParams extends BenchmarkRunsCursorIDPageParams { + /** + * The Benchmark ID to filter by. + */ + benchmark_id?: string; + + /** + * Filter by name + */ + name?: string; +} + +export interface BenchmarkRunListScenarioRunsParams extends BenchmarkRunsCursorIDPageParams { + /** + * Filter by Scenario Run state + */ + state?: 'running' | 'scoring' | 'scored' | 'completed' | 'canceled' | 'timeout' | 'failed'; +} + +BenchmarkRuns.BenchmarkRunViewsBenchmarkRunsCursorIDPage = BenchmarkRunViewsBenchmarkRunsCursorIDPage; + +export declare namespace BenchmarkRuns { + export { + type BenchmarkRunListView as BenchmarkRunListView, + type BenchmarkRunView as BenchmarkRunView, + type ScenarioRunListView as ScenarioRunListView, + BenchmarkRunViewsBenchmarkRunsCursorIDPage as BenchmarkRunViewsBenchmarkRunsCursorIDPage, + type BenchmarkRunListParams as BenchmarkRunListParams, + type BenchmarkRunListScenarioRunsParams as BenchmarkRunListScenarioRunsParams, + }; +} + +export { ScenarioRunViewsBenchmarkRunsCursorIDPage }; diff --git a/src/resources/benchmarks/benchmarks.ts b/src/resources/benchmarks/benchmarks.ts index d07a75727..0f5e15a4d 100644 --- a/src/resources/benchmarks/benchmarks.ts +++ b/src/resources/benchmarks/benchmarks.ts @@ -3,15 +3,12 @@ import { APIResource } from '../../resource'; import { isRequestOptions } from '../../core'; import * as Core from '../../core'; +import * as BenchmarkRunsAPI from '../benchmark-runs'; import * as Shared from '../shared'; import * as RunsAPI from './runs'; import { RunListParams, RunListScenarioRunsParams, Runs } from './runs'; import * as ScenariosAPI from '../scenarios/scenarios'; -import { - BenchmarkRunsCursorIDPage, - BenchmarksCursorIDPage, - type BenchmarksCursorIDPageParams, -} from '../../pagination'; +import { BenchmarksCursorIDPage, type BenchmarksCursorIDPageParams } from '../../pagination'; export class Benchmarks extends APIResource { runs: RunsAPI.Runs = new RunsAPI.Runs(this._client); @@ -119,7 +116,10 @@ export class Benchmarks extends APIResource { /** * Start a new BenchmarkRun based on the provided Benchmark. */ - startRun(body: BenchmarkStartRunParams, options?: Core.RequestOptions): Core.APIPromise { + startRun( + body: BenchmarkStartRunParams, + options?: Core.RequestOptions, + ): Core.APIPromise { return this._client.post('/v1/benchmarks/start_run', { body, ...options }); } @@ -146,8 +146,6 @@ export class Benchmarks extends APIResource { export class BenchmarkViewsBenchmarksCursorIDPage extends BenchmarksCursorIDPage {} -export class BenchmarkRunViewsBenchmarkRunsCursorIDPage extends BenchmarkRunsCursorIDPage {} - /** * BenchmarkCreateParameters contain the set of parameters to create a Benchmark. */ @@ -199,7 +197,7 @@ export interface BenchmarkRunListView { /** * List of BenchmarkRuns matching filter. */ - runs: Array; + runs: Array; total_count: number; } diff --git a/src/resources/benchmarks/index.ts b/src/resources/benchmarks/index.ts index 43efbb527..48851e79a 100644 --- a/src/resources/benchmarks/index.ts +++ b/src/resources/benchmarks/index.ts @@ -1,7 +1,6 @@ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. export { - BenchmarkRunViewsBenchmarkRunsCursorIDPage, BenchmarkViewsBenchmarksCursorIDPage, Benchmarks, type BenchmarkCreateParameters, diff --git a/src/resources/benchmarks/runs.ts b/src/resources/benchmarks/runs.ts index 18ce250d0..eec80fd59 100644 --- a/src/resources/benchmarks/runs.ts +++ b/src/resources/benchmarks/runs.ts @@ -3,8 +3,8 @@ import { APIResource } from '../../resource'; import { isRequestOptions } from '../../core'; import * as Core from '../../core'; -import * as BenchmarksAPI from './benchmarks'; -import { BenchmarkRunViewsBenchmarkRunsCursorIDPage } from './benchmarks'; +import * as BenchmarkRunsAPI from '../benchmark-runs'; +import { BenchmarkRunViewsBenchmarkRunsCursorIDPage } from '../benchmark-runs'; import * as ScenariosAPI from '../scenarios/scenarios'; import { ScenarioRunViewsBenchmarkRunsCursorIDPage } from '../scenarios/scenarios'; import { type BenchmarkRunsCursorIDPageParams } from '../../pagination'; @@ -12,25 +12,29 @@ import { type BenchmarkRunsCursorIDPageParams } from '../../pagination'; export class Runs extends APIResource { /** * Get a BenchmarkRun given ID. + * + * @deprecated */ - retrieve(id: string, options?: Core.RequestOptions): Core.APIPromise { + retrieve(id: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.get(`/v1/benchmarks/runs/${id}`, options); } /** * List all BenchmarkRuns matching filter. + * + * @deprecated */ list( query?: RunListParams, options?: Core.RequestOptions, - ): Core.PagePromise; + ): Core.PagePromise; list( options?: Core.RequestOptions, - ): Core.PagePromise; + ): Core.PagePromise; list( query: RunListParams | Core.RequestOptions = {}, options?: Core.RequestOptions, - ): Core.PagePromise { + ): Core.PagePromise { if (isRequestOptions(query)) { return this.list({}, query); } @@ -42,20 +46,26 @@ export class Runs extends APIResource { /** * Cancel a currently running Benchmark run. + * + * @deprecated */ - cancel(id: string, options?: Core.RequestOptions): Core.APIPromise { + cancel(id: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.post(`/v1/benchmarks/runs/${id}/cancel`, options); } /** * Complete a currently running BenchmarkRun. + * + * @deprecated */ - complete(id: string, options?: Core.RequestOptions): Core.APIPromise { + complete(id: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.post(`/v1/benchmarks/runs/${id}/complete`, options); } /** * List started scenario runs for a benchmark run. + * + * @deprecated */ listScenarioRuns( id: string, diff --git a/src/resources/index.ts b/src/resources/index.ts index b1a19bb20..6778a73b5 100644 --- a/src/resources/index.ts +++ b/src/resources/index.ts @@ -12,6 +12,14 @@ export { } from './agents'; export { BenchmarkRunViewsBenchmarkRunsCursorIDPage, + BenchmarkRuns, + type BenchmarkRunListView, + type BenchmarkRunView, + type ScenarioRunListView, + type BenchmarkRunListParams, + type BenchmarkRunListScenarioRunsParams, +} from './benchmark-runs'; +export { BenchmarkViewsBenchmarksCursorIDPage, Benchmarks, type BenchmarkCreateParameters, diff --git a/tests/api-resources/benchmark-runs.test.ts b/tests/api-resources/benchmark-runs.test.ts new file mode 100644 index 000000000..1ea8a5ba9 --- /dev/null +++ b/tests/api-resources/benchmark-runs.test.ts @@ -0,0 +1,131 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import Runloop from '@runloop/api-client'; +import { Response } from 'node-fetch'; + +const client = new Runloop({ + bearerToken: 'My Bearer Token', + baseURL: process.env['TEST_API_BASE_URL'] ?? 'http://127.0.0.1:4010', +}); + +describe('resource benchmarkRuns', () => { + test('retrieve', async () => { + const responsePromise = client.benchmarkRuns.retrieve('id'); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('retrieve: request options instead of params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect(client.benchmarkRuns.retrieve('id', { path: '/_stainless_unknown_path' })).rejects.toThrow( + Runloop.NotFoundError, + ); + }); + + test('list', async () => { + const responsePromise = client.benchmarkRuns.list(); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('list: request options instead of params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect(client.benchmarkRuns.list({ path: '/_stainless_unknown_path' })).rejects.toThrow( + Runloop.NotFoundError, + ); + }); + + test('list: request options and params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect( + client.benchmarkRuns.list( + { + benchmark_id: 'benchmark_id', + limit: 0, + name: 'name', + starting_after: 'starting_after', + }, + { path: '/_stainless_unknown_path' }, + ), + ).rejects.toThrow(Runloop.NotFoundError); + }); + + test('cancel', async () => { + const responsePromise = client.benchmarkRuns.cancel('id'); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('cancel: request options instead of params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect(client.benchmarkRuns.cancel('id', { path: '/_stainless_unknown_path' })).rejects.toThrow( + Runloop.NotFoundError, + ); + }); + + test('complete', async () => { + const responsePromise = client.benchmarkRuns.complete('id'); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('complete: request options instead of params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect(client.benchmarkRuns.complete('id', { path: '/_stainless_unknown_path' })).rejects.toThrow( + Runloop.NotFoundError, + ); + }); + + test('listScenarioRuns', async () => { + const responsePromise = client.benchmarkRuns.listScenarioRuns('id'); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('listScenarioRuns: request options instead of params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect( + client.benchmarkRuns.listScenarioRuns('id', { path: '/_stainless_unknown_path' }), + ).rejects.toThrow(Runloop.NotFoundError); + }); + + test('listScenarioRuns: request options and params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect( + client.benchmarkRuns.listScenarioRuns( + 'id', + { + limit: 0, + starting_after: 'starting_after', + state: 'running', + }, + { path: '/_stainless_unknown_path' }, + ), + ).rejects.toThrow(Runloop.NotFoundError); + }); +}); From 159b43f2864105b83fa47cef591a216aedeb2b7c Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Wed, 21 Jan 2026 21:50:40 +0000 Subject: [PATCH 4/7] chore: update network policy create params descriptions (#7069) --- .stats.yml | 4 ++-- src/resources/network-policies.ts | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.stats.yml b/.stats.yml index 553c2ca6f..6464e357e 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 108 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-e583f34fdcdc18499c8692e8eb8021f6163201f0f77206934c712c319a674d43.yml -openapi_spec_hash: f2fb3f7f5c1f62d3dc397cd02cd1007a +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-5359067a857aa94f69bae0d3311856be3e637da067fdc9dbf8bd26fe476efbd8.yml +openapi_spec_hash: 5227ef7c306d5226c3aee8932b2e8c6a config_hash: 42959fa2708796cc2f83937278dde733 diff --git a/src/resources/network-policies.ts b/src/resources/network-policies.ts index 2e18e70ac..23efcd876 100644 --- a/src/resources/network-policies.ts +++ b/src/resources/network-policies.ts @@ -89,18 +89,20 @@ export interface NetworkPolicyCreateParameters { name: string; /** - * If true, all egress traffic is allowed (ALLOW_ALL policy). Defaults to false. + * (Optional) If true, all egress traffic is allowed (ALLOW_ALL policy). Defaults + * to false. */ allow_all?: boolean | null; /** - * If true, allows traffic between the account's own devboxes via tunnels. Defaults - * to false. + * (Optional) If true, allows traffic between the account's own devboxes via + * tunnels. Defaults to false. If allow_all is true, this is automatically set to + * true. */ allow_devbox_to_devbox?: boolean | null; /** - * DNS-based allow list with wildcard support. Examples: ['github.com', + * (Optional) DNS-based allow list with wildcard support. Examples: ['github.com', * '*.npmjs.org']. */ allowed_hostnames?: Array | null; @@ -231,18 +233,20 @@ export interface NetworkPolicyCreateParams { name: string; /** - * If true, all egress traffic is allowed (ALLOW_ALL policy). Defaults to false. + * (Optional) If true, all egress traffic is allowed (ALLOW_ALL policy). Defaults + * to false. */ allow_all?: boolean | null; /** - * If true, allows traffic between the account's own devboxes via tunnels. Defaults - * to false. + * (Optional) If true, allows traffic between the account's own devboxes via + * tunnels. Defaults to false. If allow_all is true, this is automatically set to + * true. */ allow_devbox_to_devbox?: boolean | null; /** - * DNS-based allow list with wildcard support. Examples: ['github.com', + * (Optional) DNS-based allow list with wildcard support. Examples: ['github.com', * '*.npmjs.org']. */ allowed_hostnames?: Array | null; From d772c92acae9b124a797d204a0d1a9bdaa0eec19 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 22 Jan 2026 00:16:20 +0000 Subject: [PATCH 5/7] chore: fix stainless spec / remove deprecated benchmark/runs (#7074) --- .stats.yml | 4 +- api.md | 41 +- src/index.ts | 44 +- src/resources/benchmark-runs.ts | 14 - src/resources/benchmarks.ts | 488 ++++++++++++++- src/resources/benchmarks/benchmarks.ts | 580 ------------------ src/resources/benchmarks/index.ts | 22 - src/resources/benchmarks/runs.ts | 118 ---- src/resources/index.ts | 5 +- .../{benchmarks => }/benchmarks.test.ts | 0 tests/api-resources/benchmarks/runs.test.ts | 131 ---- 11 files changed, 523 insertions(+), 924 deletions(-) delete mode 100644 src/resources/benchmarks/benchmarks.ts delete mode 100644 src/resources/benchmarks/index.ts delete mode 100644 src/resources/benchmarks/runs.ts rename tests/api-resources/{benchmarks => }/benchmarks.test.ts (100%) delete mode 100644 tests/api-resources/benchmarks/runs.test.ts diff --git a/.stats.yml b/.stats.yml index 6464e357e..5eb10a624 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 108 +configured_endpoints: 103 openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-5359067a857aa94f69bae0d3311856be3e637da067fdc9dbf8bd26fe476efbd8.yml openapi_spec_hash: 5227ef7c306d5226c3aee8932b2e8c6a -config_hash: 42959fa2708796cc2f83937278dde733 +config_hash: cb43d4ca9e64d5a099199d6818d70539 diff --git a/api.md b/api.md index 83b439780..36cb120cf 100644 --- a/api.md +++ b/api.md @@ -15,35 +15,23 @@ Types: Types: -- BenchmarkCreateParameters -- BenchmarkRunListView -- BenchmarkRunView -- BenchmarkScenarioUpdateParameters -- BenchmarkUpdateParameters -- BenchmarkView -- ScenarioDefinitionListView -- StartBenchmarkRunParameters +- BenchmarkCreateParameters +- BenchmarkScenarioUpdateParameters +- BenchmarkUpdateParameters +- BenchmarkView +- ScenarioDefinitionListView +- StartBenchmarkRunParameters Methods: -- client.benchmarks.create({ ...params }) -> BenchmarkView -- client.benchmarks.retrieve(id) -> BenchmarkView -- client.benchmarks.update(id, { ...params }) -> BenchmarkView -- client.benchmarks.list({ ...params }) -> BenchmarkViewsBenchmarksCursorIDPage -- client.benchmarks.definitions(id, { ...params }) -> ScenarioDefinitionListView -- client.benchmarks.listPublic({ ...params }) -> BenchmarkViewsBenchmarksCursorIDPage -- client.benchmarks.startRun({ ...params }) -> BenchmarkRunView -- client.benchmarks.updateScenarios(id, { ...params }) -> BenchmarkView - -## Runs - -Methods: - -- client.benchmarks.runs.retrieve(id) -> BenchmarkRunView -- client.benchmarks.runs.list({ ...params }) -> BenchmarkRunViewsBenchmarkRunsCursorIDPage -- client.benchmarks.runs.cancel(id) -> BenchmarkRunView -- client.benchmarks.runs.complete(id) -> BenchmarkRunView -- client.benchmarks.runs.listScenarioRuns(id, { ...params }) -> ScenarioRunViewsBenchmarkRunsCursorIDPage +- client.benchmarks.create({ ...params }) -> BenchmarkView +- client.benchmarks.retrieve(id) -> BenchmarkView +- client.benchmarks.update(id, { ...params }) -> BenchmarkView +- client.benchmarks.list({ ...params }) -> BenchmarkViewsBenchmarksCursorIDPage +- client.benchmarks.definitions(id, { ...params }) -> ScenarioDefinitionListView +- client.benchmarks.listPublic({ ...params }) -> BenchmarkViewsBenchmarksCursorIDPage +- client.benchmarks.startRun({ ...params }) -> BenchmarkRunView +- client.benchmarks.updateScenarios(id, { ...params }) -> BenchmarkView # BenchmarkRuns @@ -51,7 +39,6 @@ Types: - BenchmarkRunListView - BenchmarkRunView -- ScenarioRunListView Methods: diff --git a/src/index.ts b/src/index.ts index bbce762fd..b73020579 100644 --- a/src/index.ts +++ b/src/index.ts @@ -48,8 +48,24 @@ import { BenchmarkRunView, BenchmarkRunViewsBenchmarkRunsCursorIDPage, BenchmarkRuns, - ScenarioRunListView, } from './resources/benchmark-runs'; +import { + BenchmarkCreateParameters, + BenchmarkCreateParams, + BenchmarkDefinitionsParams, + BenchmarkListParams, + BenchmarkListPublicParams, + BenchmarkScenarioUpdateParameters, + BenchmarkStartRunParams, + BenchmarkUpdateParameters, + BenchmarkUpdateParams, + BenchmarkUpdateScenariosParams, + BenchmarkView, + BenchmarkViewsBenchmarksCursorIDPage, + Benchmarks, + ScenarioDefinitionListView, + StartBenchmarkRunParameters, +} from './resources/benchmarks'; import { BlueprintBuildFromInspectionParameters, BlueprintBuildLog, @@ -121,25 +137,6 @@ import { SecretView, Secrets, } from './resources/secrets'; -import { - BenchmarkCreateParameters, - BenchmarkCreateParams, - BenchmarkDefinitionsParams, - BenchmarkListParams, - BenchmarkListPublicParams, - BenchmarkRunListView as BenchmarksAPIBenchmarkRunListView, - BenchmarkRunView as BenchmarksAPIBenchmarkRunView, - BenchmarkScenarioUpdateParameters, - BenchmarkStartRunParams, - BenchmarkUpdateParameters, - BenchmarkUpdateParams, - BenchmarkUpdateScenariosParams, - BenchmarkView, - BenchmarkViewsBenchmarksCursorIDPage, - Benchmarks, - ScenarioDefinitionListView, - StartBenchmarkRunParameters, -} from './resources/benchmarks/benchmarks'; import { DevboxAsyncExecutionDetailView, DevboxCreateParams, @@ -185,7 +182,7 @@ import { ScenarioEnvironment, ScenarioListParams, ScenarioListPublicParams, - ScenarioRunListView as ScenariosAPIScenarioRunListView, + ScenarioRunListView, ScenarioRunView, ScenarioStartRunParams, ScenarioUpdateParameters, @@ -478,8 +475,6 @@ export declare namespace Runloop { export { Benchmarks as Benchmarks, type BenchmarkCreateParameters as BenchmarkCreateParameters, - type BenchmarksAPIBenchmarkRunListView as BenchmarkRunListView, - type BenchmarksAPIBenchmarkRunView as BenchmarkRunView, type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters, type BenchmarkUpdateParameters as BenchmarkUpdateParameters, type BenchmarkView as BenchmarkView, @@ -499,7 +494,6 @@ export declare namespace Runloop { BenchmarkRuns as BenchmarkRuns, type BenchmarkRunListView as BenchmarkRunListView, type BenchmarkRunView as BenchmarkRunView, - type ScenarioRunListView as ScenarioRunListView, BenchmarkRunViewsBenchmarkRunsCursorIDPage as BenchmarkRunViewsBenchmarkRunsCursorIDPage, type BenchmarkRunListParams as BenchmarkRunListParams, type BenchmarkRunListScenarioRunsParams as BenchmarkRunListScenarioRunsParams, @@ -578,7 +572,7 @@ export declare namespace Runloop { type InputContextUpdate as InputContextUpdate, type ScenarioCreateParameters as ScenarioCreateParameters, type ScenarioEnvironment as ScenarioEnvironment, - type ScenariosAPIScenarioRunListView as ScenarioRunListView, + type ScenarioRunListView as ScenarioRunListView, type ScenarioRunView as ScenarioRunView, type ScenarioUpdateParameters as ScenarioUpdateParameters, type ScenarioView as ScenarioView, diff --git a/src/resources/benchmark-runs.ts b/src/resources/benchmark-runs.ts index 1e83138be..7d51c5195 100644 --- a/src/resources/benchmark-runs.ts +++ b/src/resources/benchmark-runs.ts @@ -160,19 +160,6 @@ export interface BenchmarkRunView { secrets_provided?: { [key: string]: string } | null; } -export interface ScenarioRunListView { - has_more: boolean; - - remaining_count: number; - - /** - * List of ScenarioRuns matching filter. - */ - runs: Array; - - total_count: number; -} - export interface BenchmarkRunListParams extends BenchmarkRunsCursorIDPageParams { /** * The Benchmark ID to filter by. @@ -198,7 +185,6 @@ export declare namespace BenchmarkRuns { export { type BenchmarkRunListView as BenchmarkRunListView, type BenchmarkRunView as BenchmarkRunView, - type ScenarioRunListView as ScenarioRunListView, BenchmarkRunViewsBenchmarkRunsCursorIDPage as BenchmarkRunViewsBenchmarkRunsCursorIDPage, type BenchmarkRunListParams as BenchmarkRunListParams, type BenchmarkRunListScenarioRunsParams as BenchmarkRunListScenarioRunsParams, diff --git a/src/resources/benchmarks.ts b/src/resources/benchmarks.ts index f9ae0773d..c0d264c9d 100644 --- a/src/resources/benchmarks.ts +++ b/src/resources/benchmarks.ts @@ -1,3 +1,489 @@ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -export * from './benchmarks/index'; +import { APIResource } from '../resource'; +import { isRequestOptions } from '../core'; +import * as Core from '../core'; +import * as BenchmarkRunsAPI from './benchmark-runs'; +import * as Shared from './shared'; +import * as ScenariosAPI from './scenarios/scenarios'; +import { BenchmarksCursorIDPage, type BenchmarksCursorIDPageParams } from '../pagination'; + +export class Benchmarks extends APIResource { + /** + * Create a Benchmark with a set of Scenarios. + */ + create(body: BenchmarkCreateParams, options?: Core.RequestOptions): Core.APIPromise { + return this._client.post('/v1/benchmarks', { body, ...options }); + } + + /** + * Get a previously created Benchmark. + */ + retrieve(id: string, options?: Core.RequestOptions): Core.APIPromise { + return this._client.get(`/v1/benchmarks/${id}`, options); + } + + /** + * Update a Benchmark. Fields that are null will preserve the existing value. + * Fields that are provided (including empty values) will replace the existing + * value entirely. + */ + update( + id: string, + body?: BenchmarkUpdateParams, + options?: Core.RequestOptions, + ): Core.APIPromise; + update(id: string, options?: Core.RequestOptions): Core.APIPromise; + update( + id: string, + body: BenchmarkUpdateParams | Core.RequestOptions = {}, + options?: Core.RequestOptions, + ): Core.APIPromise { + if (isRequestOptions(body)) { + return this.update(id, {}, body); + } + return this._client.post(`/v1/benchmarks/${id}`, { body, ...options }); + } + + /** + * List all Benchmarks matching filter. + */ + list( + query?: BenchmarkListParams, + options?: Core.RequestOptions, + ): Core.PagePromise; + list(options?: Core.RequestOptions): Core.PagePromise; + list( + query: BenchmarkListParams | Core.RequestOptions = {}, + options?: Core.RequestOptions, + ): Core.PagePromise { + if (isRequestOptions(query)) { + return this.list({}, query); + } + return this._client.getAPIList('/v1/benchmarks', BenchmarkViewsBenchmarksCursorIDPage, { + query, + ...options, + }); + } + + /** + * Get scenario definitions for a previously created Benchmark. + */ + definitions( + id: string, + query?: BenchmarkDefinitionsParams, + options?: Core.RequestOptions, + ): Core.APIPromise; + definitions(id: string, options?: Core.RequestOptions): Core.APIPromise; + definitions( + id: string, + query: BenchmarkDefinitionsParams | Core.RequestOptions = {}, + options?: Core.RequestOptions, + ): Core.APIPromise { + if (isRequestOptions(query)) { + return this.definitions(id, {}, query); + } + return this._client.get(`/v1/benchmarks/${id}/definitions`, { query, ...options }); + } + + /** + * List all public benchmarks matching filter. + */ + listPublic( + query?: BenchmarkListPublicParams, + options?: Core.RequestOptions, + ): Core.PagePromise; + listPublic( + options?: Core.RequestOptions, + ): Core.PagePromise; + listPublic( + query: BenchmarkListPublicParams | Core.RequestOptions = {}, + options?: Core.RequestOptions, + ): Core.PagePromise { + if (isRequestOptions(query)) { + return this.listPublic({}, query); + } + return this._client.getAPIList('/v1/benchmarks/list_public', BenchmarkViewsBenchmarksCursorIDPage, { + query, + ...options, + }); + } + + /** + * Start a new BenchmarkRun based on the provided Benchmark. + */ + startRun( + body: BenchmarkStartRunParams, + options?: Core.RequestOptions, + ): Core.APIPromise { + return this._client.post('/v1/benchmarks/start_run', { body, ...options }); + } + + /** + * Add and/or remove Scenario IDs from an existing Benchmark. + */ + updateScenarios( + id: string, + body?: BenchmarkUpdateScenariosParams, + options?: Core.RequestOptions, + ): Core.APIPromise; + updateScenarios(id: string, options?: Core.RequestOptions): Core.APIPromise; + updateScenarios( + id: string, + body: BenchmarkUpdateScenariosParams | Core.RequestOptions = {}, + options?: Core.RequestOptions, + ): Core.APIPromise { + if (isRequestOptions(body)) { + return this.updateScenarios(id, {}, body); + } + return this._client.post(`/v1/benchmarks/${id}/scenarios`, { body, ...options }); + } +} + +export class BenchmarkViewsBenchmarksCursorIDPage extends BenchmarksCursorIDPage {} + +/** + * BenchmarkCreateParameters contain the set of parameters to create a Benchmark. + */ +export interface BenchmarkCreateParameters { + /** + * The unique name of the Benchmark. + */ + name: string; + + /** + * Attribution information for the benchmark. + */ + attribution?: string | null; + + /** + * Detailed description of the benchmark. + */ + description?: string | null; + + /** + * User defined metadata to attach to the benchmark. + */ + metadata?: { [key: string]: string } | null; + + /** + * Environment variables required to run the benchmark. If any required variables + * are not supplied, the benchmark will fail to start. + */ + required_environment_variables?: Array | null; + + /** + * Secrets required to run the benchmark with (environment variable name will be + * mapped to the your user secret by name). If any of these secrets are not + * provided or the mapping is incorrect, the benchmark will fail to start. + */ + required_secret_names?: Array; + + /** + * The Scenario IDs that make up the Benchmark. + */ + scenario_ids?: Array | null; +} + +export interface BenchmarkScenarioUpdateParameters { + /** + * Scenario IDs to add to the Benchmark. + */ + scenarios_to_add?: Array | null; + + /** + * Scenario IDs to remove from the Benchmark. + */ + scenarios_to_remove?: Array | null; +} + +/** + * BenchmarkUpdateParameters contain the set of parameters to update a Benchmark. + * All fields are optional - null fields preserve existing values, provided fields + * replace entirely. + */ +export interface BenchmarkUpdateParameters { + /** + * Attribution information for the benchmark. Pass in empty string to clear. + */ + attribution?: string | null; + + /** + * Detailed description of the benchmark. Pass in empty string to clear. + */ + description?: string | null; + + /** + * User defined metadata to attach to the benchmark. Pass in empty map to clear. + */ + metadata?: { [key: string]: string } | null; + + /** + * The unique name of the Benchmark. Cannot be blank. + */ + name?: string | null; + + /** + * Environment variables required to run the benchmark. If any required variables + * are not supplied, the benchmark will fail to start. Pass in empty list to clear. + */ + required_environment_variables?: Array | null; + + /** + * Secrets required to run the benchmark with (environment variable name will be + * mapped to the your user secret by name). If any of these secrets are not + * provided or the mapping is incorrect, the benchmark will fail to start. Pass in + * empty list to clear. + */ + required_secret_names?: Array | null; + + /** + * The Scenario IDs that make up the Benchmark. Pass in empty list to clear. + */ + scenario_ids?: Array | null; +} + +/** + * A BenchmarkDefinitionView represents a grouped set of Scenarios that together + * form a Benchmark. + */ +export interface BenchmarkView { + /** + * The ID of the Benchmark. + */ + id: string; + + /** + * User defined metadata to attach to the benchmark for organization. + */ + metadata: { [key: string]: string }; + + /** + * The name of the Benchmark. + */ + name: string; + + /** + * List of Scenario IDs that make up the benchmark. + */ + scenarioIds: Array; + + /** + * Attribution information for the benchmark. + */ + attribution?: string; + + /** + * Detailed description of the benchmark. + */ + description?: string; + + /** + * Whether this benchmark is public. + */ + is_public?: boolean; + + /** + * Required environment variables used to run the benchmark. If any required + * environment variables are missing, the benchmark will fail to start. + */ + required_environment_variables?: Array; + + /** + * Required secrets used to run the benchmark. If any required secrets are missing, + * the benchmark will fail to start. + */ + required_secret_names?: Array; +} + +export interface ScenarioDefinitionListView { + has_more: boolean; + + remaining_count: number; + + /** + * List of Scenarios matching filter. + */ + scenarios: Array; + + total_count: number; +} + +export interface StartBenchmarkRunParameters { + /** + * ID of the Benchmark to run. + */ + benchmark_id: string; + + /** + * User defined metadata to attach to the benchmark run for organization. + */ + metadata?: { [key: string]: string } | null; + + /** + * Display name of the run. + */ + run_name?: string | null; + + /** + * Runtime configuration to use for this benchmark run + */ + runProfile?: Shared.RunProfile | null; +} + +export interface BenchmarkCreateParams { + /** + * The unique name of the Benchmark. + */ + name: string; + + /** + * Attribution information for the benchmark. + */ + attribution?: string | null; + + /** + * Detailed description of the benchmark. + */ + description?: string | null; + + /** + * User defined metadata to attach to the benchmark. + */ + metadata?: { [key: string]: string } | null; + + /** + * Environment variables required to run the benchmark. If any required variables + * are not supplied, the benchmark will fail to start. + */ + required_environment_variables?: Array | null; + + /** + * Secrets required to run the benchmark with (environment variable name will be + * mapped to the your user secret by name). If any of these secrets are not + * provided or the mapping is incorrect, the benchmark will fail to start. + */ + required_secret_names?: Array; + + /** + * The Scenario IDs that make up the Benchmark. + */ + scenario_ids?: Array | null; +} + +export interface BenchmarkUpdateParams { + /** + * Attribution information for the benchmark. Pass in empty string to clear. + */ + attribution?: string | null; + + /** + * Detailed description of the benchmark. Pass in empty string to clear. + */ + description?: string | null; + + /** + * User defined metadata to attach to the benchmark. Pass in empty map to clear. + */ + metadata?: { [key: string]: string } | null; + + /** + * The unique name of the Benchmark. Cannot be blank. + */ + name?: string | null; + + /** + * Environment variables required to run the benchmark. If any required variables + * are not supplied, the benchmark will fail to start. Pass in empty list to clear. + */ + required_environment_variables?: Array | null; + + /** + * Secrets required to run the benchmark with (environment variable name will be + * mapped to the your user secret by name). If any of these secrets are not + * provided or the mapping is incorrect, the benchmark will fail to start. Pass in + * empty list to clear. + */ + required_secret_names?: Array | null; + + /** + * The Scenario IDs that make up the Benchmark. Pass in empty list to clear. + */ + scenario_ids?: Array | null; +} + +export interface BenchmarkListParams extends BenchmarksCursorIDPageParams { + /** + * Filter by name + */ + name?: string; +} + +export interface BenchmarkDefinitionsParams { + /** + * The limit of items to return. Default is 20. Max is 5000. + */ + limit?: number; + + /** + * Load the next page of data starting after the item with the given ID. + */ + starting_after?: string; +} + +export interface BenchmarkListPublicParams extends BenchmarksCursorIDPageParams {} + +export interface BenchmarkStartRunParams { + /** + * ID of the Benchmark to run. + */ + benchmark_id: string; + + /** + * User defined metadata to attach to the benchmark run for organization. + */ + metadata?: { [key: string]: string } | null; + + /** + * Display name of the run. + */ + run_name?: string | null; + + /** + * Runtime configuration to use for this benchmark run + */ + runProfile?: Shared.RunProfile | null; +} + +export interface BenchmarkUpdateScenariosParams { + /** + * Scenario IDs to add to the Benchmark. + */ + scenarios_to_add?: Array | null; + + /** + * Scenario IDs to remove from the Benchmark. + */ + scenarios_to_remove?: Array | null; +} + +Benchmarks.BenchmarkViewsBenchmarksCursorIDPage = BenchmarkViewsBenchmarksCursorIDPage; + +export declare namespace Benchmarks { + export { + type BenchmarkCreateParameters as BenchmarkCreateParameters, + type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters, + type BenchmarkUpdateParameters as BenchmarkUpdateParameters, + type BenchmarkView as BenchmarkView, + type ScenarioDefinitionListView as ScenarioDefinitionListView, + type StartBenchmarkRunParameters as StartBenchmarkRunParameters, + BenchmarkViewsBenchmarksCursorIDPage as BenchmarkViewsBenchmarksCursorIDPage, + type BenchmarkCreateParams as BenchmarkCreateParams, + type BenchmarkUpdateParams as BenchmarkUpdateParams, + type BenchmarkListParams as BenchmarkListParams, + type BenchmarkDefinitionsParams as BenchmarkDefinitionsParams, + type BenchmarkListPublicParams as BenchmarkListPublicParams, + type BenchmarkStartRunParams as BenchmarkStartRunParams, + type BenchmarkUpdateScenariosParams as BenchmarkUpdateScenariosParams, + }; +} diff --git a/src/resources/benchmarks/benchmarks.ts b/src/resources/benchmarks/benchmarks.ts deleted file mode 100644 index 0f5e15a4d..000000000 --- a/src/resources/benchmarks/benchmarks.ts +++ /dev/null @@ -1,580 +0,0 @@ -// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -import { APIResource } from '../../resource'; -import { isRequestOptions } from '../../core'; -import * as Core from '../../core'; -import * as BenchmarkRunsAPI from '../benchmark-runs'; -import * as Shared from '../shared'; -import * as RunsAPI from './runs'; -import { RunListParams, RunListScenarioRunsParams, Runs } from './runs'; -import * as ScenariosAPI from '../scenarios/scenarios'; -import { BenchmarksCursorIDPage, type BenchmarksCursorIDPageParams } from '../../pagination'; - -export class Benchmarks extends APIResource { - runs: RunsAPI.Runs = new RunsAPI.Runs(this._client); - - /** - * Create a Benchmark with a set of Scenarios. - */ - create(body: BenchmarkCreateParams, options?: Core.RequestOptions): Core.APIPromise { - return this._client.post('/v1/benchmarks', { body, ...options }); - } - - /** - * Get a previously created Benchmark. - */ - retrieve(id: string, options?: Core.RequestOptions): Core.APIPromise { - return this._client.get(`/v1/benchmarks/${id}`, options); - } - - /** - * Update a Benchmark. Fields that are null will preserve the existing value. - * Fields that are provided (including empty values) will replace the existing - * value entirely. - */ - update( - id: string, - body?: BenchmarkUpdateParams, - options?: Core.RequestOptions, - ): Core.APIPromise; - update(id: string, options?: Core.RequestOptions): Core.APIPromise; - update( - id: string, - body: BenchmarkUpdateParams | Core.RequestOptions = {}, - options?: Core.RequestOptions, - ): Core.APIPromise { - if (isRequestOptions(body)) { - return this.update(id, {}, body); - } - return this._client.post(`/v1/benchmarks/${id}`, { body, ...options }); - } - - /** - * List all Benchmarks matching filter. - */ - list( - query?: BenchmarkListParams, - options?: Core.RequestOptions, - ): Core.PagePromise; - list(options?: Core.RequestOptions): Core.PagePromise; - list( - query: BenchmarkListParams | Core.RequestOptions = {}, - options?: Core.RequestOptions, - ): Core.PagePromise { - if (isRequestOptions(query)) { - return this.list({}, query); - } - return this._client.getAPIList('/v1/benchmarks', BenchmarkViewsBenchmarksCursorIDPage, { - query, - ...options, - }); - } - - /** - * Get scenario definitions for a previously created Benchmark. - */ - definitions( - id: string, - query?: BenchmarkDefinitionsParams, - options?: Core.RequestOptions, - ): Core.APIPromise; - definitions(id: string, options?: Core.RequestOptions): Core.APIPromise; - definitions( - id: string, - query: BenchmarkDefinitionsParams | Core.RequestOptions = {}, - options?: Core.RequestOptions, - ): Core.APIPromise { - if (isRequestOptions(query)) { - return this.definitions(id, {}, query); - } - return this._client.get(`/v1/benchmarks/${id}/definitions`, { query, ...options }); - } - - /** - * List all public benchmarks matching filter. - */ - listPublic( - query?: BenchmarkListPublicParams, - options?: Core.RequestOptions, - ): Core.PagePromise; - listPublic( - options?: Core.RequestOptions, - ): Core.PagePromise; - listPublic( - query: BenchmarkListPublicParams | Core.RequestOptions = {}, - options?: Core.RequestOptions, - ): Core.PagePromise { - if (isRequestOptions(query)) { - return this.listPublic({}, query); - } - return this._client.getAPIList('/v1/benchmarks/list_public', BenchmarkViewsBenchmarksCursorIDPage, { - query, - ...options, - }); - } - - /** - * Start a new BenchmarkRun based on the provided Benchmark. - */ - startRun( - body: BenchmarkStartRunParams, - options?: Core.RequestOptions, - ): Core.APIPromise { - return this._client.post('/v1/benchmarks/start_run', { body, ...options }); - } - - /** - * Add and/or remove Scenario IDs from an existing Benchmark. - */ - updateScenarios( - id: string, - body?: BenchmarkUpdateScenariosParams, - options?: Core.RequestOptions, - ): Core.APIPromise; - updateScenarios(id: string, options?: Core.RequestOptions): Core.APIPromise; - updateScenarios( - id: string, - body: BenchmarkUpdateScenariosParams | Core.RequestOptions = {}, - options?: Core.RequestOptions, - ): Core.APIPromise { - if (isRequestOptions(body)) { - return this.updateScenarios(id, {}, body); - } - return this._client.post(`/v1/benchmarks/${id}/scenarios`, { body, ...options }); - } -} - -export class BenchmarkViewsBenchmarksCursorIDPage extends BenchmarksCursorIDPage {} - -/** - * BenchmarkCreateParameters contain the set of parameters to create a Benchmark. - */ -export interface BenchmarkCreateParameters { - /** - * The unique name of the Benchmark. - */ - name: string; - - /** - * Attribution information for the benchmark. - */ - attribution?: string | null; - - /** - * Detailed description of the benchmark. - */ - description?: string | null; - - /** - * User defined metadata to attach to the benchmark. - */ - metadata?: { [key: string]: string } | null; - - /** - * Environment variables required to run the benchmark. If any required variables - * are not supplied, the benchmark will fail to start. - */ - required_environment_variables?: Array | null; - - /** - * Secrets required to run the benchmark with (environment variable name will be - * mapped to the your user secret by name). If any of these secrets are not - * provided or the mapping is incorrect, the benchmark will fail to start. - */ - required_secret_names?: Array; - - /** - * The Scenario IDs that make up the Benchmark. - */ - scenario_ids?: Array | null; -} - -export interface BenchmarkRunListView { - has_more: boolean; - - remaining_count: number; - - /** - * List of BenchmarkRuns matching filter. - */ - runs: Array; - - total_count: number; -} - -/** - * A BenchmarkRunView represents a run of a complete set of Scenarios, organized - * under a Benchmark or created by a BenchmarkJob. - */ -export interface BenchmarkRunView { - /** - * The ID of the BenchmarkRun. - */ - id: string; - - /** - * User defined metadata to attach to the benchmark run for organization. - */ - metadata: { [key: string]: string }; - - /** - * The time the benchmark run execution started (Unix timestamp milliseconds). - */ - start_time_ms: number; - - /** - * The state of the BenchmarkRun. - */ - state: 'running' | 'canceled' | 'completed'; - - /** - * The ID of the Benchmark definition. Present if run was created from a benchmark - * definition. - */ - benchmark_id?: string | null; - - /** - * The duration for the BenchmarkRun to complete. - */ - duration_ms?: number | null; - - /** - * Environment variables used to run the benchmark. - */ - environment_variables?: { [key: string]: string } | null; - - /** - * The name of the BenchmarkRun. - */ - name?: string | null; - - /** - * Purpose of the run. - */ - purpose?: string | null; - - /** - * The final score across the BenchmarkRun, present once completed. Calculated as - * sum of scenario scores / number of scenario runs. - */ - score?: number | null; - - /** - * User secrets used to run the benchmark. Example: {"DB_PASS": - * "DATABASE_PASSWORD"} would set the environment variable 'DB_PASS' on all - * scenario devboxes to the value of the secret 'DATABASE_PASSWORD'. - */ - secrets_provided?: { [key: string]: string } | null; -} - -export interface BenchmarkScenarioUpdateParameters { - /** - * Scenario IDs to add to the Benchmark. - */ - scenarios_to_add?: Array | null; - - /** - * Scenario IDs to remove from the Benchmark. - */ - scenarios_to_remove?: Array | null; -} - -/** - * BenchmarkUpdateParameters contain the set of parameters to update a Benchmark. - * All fields are optional - null fields preserve existing values, provided fields - * replace entirely. - */ -export interface BenchmarkUpdateParameters { - /** - * Attribution information for the benchmark. Pass in empty string to clear. - */ - attribution?: string | null; - - /** - * Detailed description of the benchmark. Pass in empty string to clear. - */ - description?: string | null; - - /** - * User defined metadata to attach to the benchmark. Pass in empty map to clear. - */ - metadata?: { [key: string]: string } | null; - - /** - * The unique name of the Benchmark. Cannot be blank. - */ - name?: string | null; - - /** - * Environment variables required to run the benchmark. If any required variables - * are not supplied, the benchmark will fail to start. Pass in empty list to clear. - */ - required_environment_variables?: Array | null; - - /** - * Secrets required to run the benchmark with (environment variable name will be - * mapped to the your user secret by name). If any of these secrets are not - * provided or the mapping is incorrect, the benchmark will fail to start. Pass in - * empty list to clear. - */ - required_secret_names?: Array | null; - - /** - * The Scenario IDs that make up the Benchmark. Pass in empty list to clear. - */ - scenario_ids?: Array | null; -} - -/** - * A BenchmarkDefinitionView represents a grouped set of Scenarios that together - * form a Benchmark. - */ -export interface BenchmarkView { - /** - * The ID of the Benchmark. - */ - id: string; - - /** - * User defined metadata to attach to the benchmark for organization. - */ - metadata: { [key: string]: string }; - - /** - * The name of the Benchmark. - */ - name: string; - - /** - * List of Scenario IDs that make up the benchmark. - */ - scenarioIds: Array; - - /** - * Attribution information for the benchmark. - */ - attribution?: string; - - /** - * Detailed description of the benchmark. - */ - description?: string; - - /** - * Whether this benchmark is public. - */ - is_public?: boolean; - - /** - * Required environment variables used to run the benchmark. If any required - * environment variables are missing, the benchmark will fail to start. - */ - required_environment_variables?: Array; - - /** - * Required secrets used to run the benchmark. If any required secrets are missing, - * the benchmark will fail to start. - */ - required_secret_names?: Array; -} - -export interface ScenarioDefinitionListView { - has_more: boolean; - - remaining_count: number; - - /** - * List of Scenarios matching filter. - */ - scenarios: Array; - - total_count: number; -} - -export interface StartBenchmarkRunParameters { - /** - * ID of the Benchmark to run. - */ - benchmark_id: string; - - /** - * User defined metadata to attach to the benchmark run for organization. - */ - metadata?: { [key: string]: string } | null; - - /** - * Display name of the run. - */ - run_name?: string | null; - - /** - * Runtime configuration to use for this benchmark run - */ - runProfile?: Shared.RunProfile | null; -} - -export interface BenchmarkCreateParams { - /** - * The unique name of the Benchmark. - */ - name: string; - - /** - * Attribution information for the benchmark. - */ - attribution?: string | null; - - /** - * Detailed description of the benchmark. - */ - description?: string | null; - - /** - * User defined metadata to attach to the benchmark. - */ - metadata?: { [key: string]: string } | null; - - /** - * Environment variables required to run the benchmark. If any required variables - * are not supplied, the benchmark will fail to start. - */ - required_environment_variables?: Array | null; - - /** - * Secrets required to run the benchmark with (environment variable name will be - * mapped to the your user secret by name). If any of these secrets are not - * provided or the mapping is incorrect, the benchmark will fail to start. - */ - required_secret_names?: Array; - - /** - * The Scenario IDs that make up the Benchmark. - */ - scenario_ids?: Array | null; -} - -export interface BenchmarkUpdateParams { - /** - * Attribution information for the benchmark. Pass in empty string to clear. - */ - attribution?: string | null; - - /** - * Detailed description of the benchmark. Pass in empty string to clear. - */ - description?: string | null; - - /** - * User defined metadata to attach to the benchmark. Pass in empty map to clear. - */ - metadata?: { [key: string]: string } | null; - - /** - * The unique name of the Benchmark. Cannot be blank. - */ - name?: string | null; - - /** - * Environment variables required to run the benchmark. If any required variables - * are not supplied, the benchmark will fail to start. Pass in empty list to clear. - */ - required_environment_variables?: Array | null; - - /** - * Secrets required to run the benchmark with (environment variable name will be - * mapped to the your user secret by name). If any of these secrets are not - * provided or the mapping is incorrect, the benchmark will fail to start. Pass in - * empty list to clear. - */ - required_secret_names?: Array | null; - - /** - * The Scenario IDs that make up the Benchmark. Pass in empty list to clear. - */ - scenario_ids?: Array | null; -} - -export interface BenchmarkListParams extends BenchmarksCursorIDPageParams { - /** - * Filter by name - */ - name?: string; -} - -export interface BenchmarkDefinitionsParams { - /** - * The limit of items to return. Default is 20. Max is 5000. - */ - limit?: number; - - /** - * Load the next page of data starting after the item with the given ID. - */ - starting_after?: string; -} - -export interface BenchmarkListPublicParams extends BenchmarksCursorIDPageParams {} - -export interface BenchmarkStartRunParams { - /** - * ID of the Benchmark to run. - */ - benchmark_id: string; - - /** - * User defined metadata to attach to the benchmark run for organization. - */ - metadata?: { [key: string]: string } | null; - - /** - * Display name of the run. - */ - run_name?: string | null; - - /** - * Runtime configuration to use for this benchmark run - */ - runProfile?: Shared.RunProfile | null; -} - -export interface BenchmarkUpdateScenariosParams { - /** - * Scenario IDs to add to the Benchmark. - */ - scenarios_to_add?: Array | null; - - /** - * Scenario IDs to remove from the Benchmark. - */ - scenarios_to_remove?: Array | null; -} - -Benchmarks.BenchmarkViewsBenchmarksCursorIDPage = BenchmarkViewsBenchmarksCursorIDPage; -Benchmarks.Runs = Runs; - -export declare namespace Benchmarks { - export { - type BenchmarkCreateParameters as BenchmarkCreateParameters, - type BenchmarkRunListView as BenchmarkRunListView, - type BenchmarkRunView as BenchmarkRunView, - type BenchmarkScenarioUpdateParameters as BenchmarkScenarioUpdateParameters, - type BenchmarkUpdateParameters as BenchmarkUpdateParameters, - type BenchmarkView as BenchmarkView, - type ScenarioDefinitionListView as ScenarioDefinitionListView, - type StartBenchmarkRunParameters as StartBenchmarkRunParameters, - BenchmarkViewsBenchmarksCursorIDPage as BenchmarkViewsBenchmarksCursorIDPage, - type BenchmarkCreateParams as BenchmarkCreateParams, - type BenchmarkUpdateParams as BenchmarkUpdateParams, - type BenchmarkListParams as BenchmarkListParams, - type BenchmarkDefinitionsParams as BenchmarkDefinitionsParams, - type BenchmarkListPublicParams as BenchmarkListPublicParams, - type BenchmarkStartRunParams as BenchmarkStartRunParams, - type BenchmarkUpdateScenariosParams as BenchmarkUpdateScenariosParams, - }; - - export { - Runs as Runs, - type RunListParams as RunListParams, - type RunListScenarioRunsParams as RunListScenarioRunsParams, - }; -} diff --git a/src/resources/benchmarks/index.ts b/src/resources/benchmarks/index.ts deleted file mode 100644 index 48851e79a..000000000 --- a/src/resources/benchmarks/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -export { - BenchmarkViewsBenchmarksCursorIDPage, - Benchmarks, - type BenchmarkCreateParameters, - type BenchmarkRunListView, - type BenchmarkRunView, - type BenchmarkScenarioUpdateParameters, - type BenchmarkUpdateParameters, - type BenchmarkView, - type ScenarioDefinitionListView, - type StartBenchmarkRunParameters, - type BenchmarkCreateParams, - type BenchmarkUpdateParams, - type BenchmarkListParams, - type BenchmarkDefinitionsParams, - type BenchmarkListPublicParams, - type BenchmarkStartRunParams, - type BenchmarkUpdateScenariosParams, -} from './benchmarks'; -export { Runs, type RunListParams, type RunListScenarioRunsParams } from './runs'; diff --git a/src/resources/benchmarks/runs.ts b/src/resources/benchmarks/runs.ts deleted file mode 100644 index eec80fd59..000000000 --- a/src/resources/benchmarks/runs.ts +++ /dev/null @@ -1,118 +0,0 @@ -// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -import { APIResource } from '../../resource'; -import { isRequestOptions } from '../../core'; -import * as Core from '../../core'; -import * as BenchmarkRunsAPI from '../benchmark-runs'; -import { BenchmarkRunViewsBenchmarkRunsCursorIDPage } from '../benchmark-runs'; -import * as ScenariosAPI from '../scenarios/scenarios'; -import { ScenarioRunViewsBenchmarkRunsCursorIDPage } from '../scenarios/scenarios'; -import { type BenchmarkRunsCursorIDPageParams } from '../../pagination'; - -export class Runs extends APIResource { - /** - * Get a BenchmarkRun given ID. - * - * @deprecated - */ - retrieve(id: string, options?: Core.RequestOptions): Core.APIPromise { - return this._client.get(`/v1/benchmarks/runs/${id}`, options); - } - - /** - * List all BenchmarkRuns matching filter. - * - * @deprecated - */ - list( - query?: RunListParams, - options?: Core.RequestOptions, - ): Core.PagePromise; - list( - options?: Core.RequestOptions, - ): Core.PagePromise; - list( - query: RunListParams | Core.RequestOptions = {}, - options?: Core.RequestOptions, - ): Core.PagePromise { - if (isRequestOptions(query)) { - return this.list({}, query); - } - return this._client.getAPIList('/v1/benchmarks/runs', BenchmarkRunViewsBenchmarkRunsCursorIDPage, { - query, - ...options, - }); - } - - /** - * Cancel a currently running Benchmark run. - * - * @deprecated - */ - cancel(id: string, options?: Core.RequestOptions): Core.APIPromise { - return this._client.post(`/v1/benchmarks/runs/${id}/cancel`, options); - } - - /** - * Complete a currently running BenchmarkRun. - * - * @deprecated - */ - complete(id: string, options?: Core.RequestOptions): Core.APIPromise { - return this._client.post(`/v1/benchmarks/runs/${id}/complete`, options); - } - - /** - * List started scenario runs for a benchmark run. - * - * @deprecated - */ - listScenarioRuns( - id: string, - query?: RunListScenarioRunsParams, - options?: Core.RequestOptions, - ): Core.PagePromise; - listScenarioRuns( - id: string, - options?: Core.RequestOptions, - ): Core.PagePromise; - listScenarioRuns( - id: string, - query: RunListScenarioRunsParams | Core.RequestOptions = {}, - options?: Core.RequestOptions, - ): Core.PagePromise { - if (isRequestOptions(query)) { - return this.listScenarioRuns(id, {}, query); - } - return this._client.getAPIList( - `/v1/benchmarks/runs/${id}/scenario_runs`, - ScenarioRunViewsBenchmarkRunsCursorIDPage, - { query, ...options }, - ); - } -} - -export interface RunListParams extends BenchmarkRunsCursorIDPageParams { - /** - * The Benchmark ID to filter by. - */ - benchmark_id?: string; - - /** - * Filter by name - */ - name?: string; -} - -export interface RunListScenarioRunsParams extends BenchmarkRunsCursorIDPageParams { - /** - * Filter by Scenario Run state - */ - state?: 'running' | 'scoring' | 'scored' | 'completed' | 'canceled' | 'timeout' | 'failed'; -} - -export declare namespace Runs { - export { type RunListParams as RunListParams, type RunListScenarioRunsParams as RunListScenarioRunsParams }; -} - -export { BenchmarkRunViewsBenchmarkRunsCursorIDPage, ScenarioRunViewsBenchmarkRunsCursorIDPage }; diff --git a/src/resources/index.ts b/src/resources/index.ts index 6778a73b5..1afde0ca7 100644 --- a/src/resources/index.ts +++ b/src/resources/index.ts @@ -15,7 +15,6 @@ export { BenchmarkRuns, type BenchmarkRunListView, type BenchmarkRunView, - type ScenarioRunListView, type BenchmarkRunListParams, type BenchmarkRunListScenarioRunsParams, } from './benchmark-runs'; @@ -23,8 +22,6 @@ export { BenchmarkViewsBenchmarksCursorIDPage, Benchmarks, type BenchmarkCreateParameters, - type BenchmarkRunListView, - type BenchmarkRunView, type BenchmarkScenarioUpdateParameters, type BenchmarkUpdateParameters, type BenchmarkView, @@ -37,7 +34,7 @@ export { type BenchmarkListPublicParams, type BenchmarkStartRunParams, type BenchmarkUpdateScenariosParams, -} from './benchmarks/benchmarks'; +} from './benchmarks'; export { BlueprintViewsBlueprintsCursorIDPage, Blueprints, diff --git a/tests/api-resources/benchmarks/benchmarks.test.ts b/tests/api-resources/benchmarks.test.ts similarity index 100% rename from tests/api-resources/benchmarks/benchmarks.test.ts rename to tests/api-resources/benchmarks.test.ts diff --git a/tests/api-resources/benchmarks/runs.test.ts b/tests/api-resources/benchmarks/runs.test.ts deleted file mode 100644 index f60726857..000000000 --- a/tests/api-resources/benchmarks/runs.test.ts +++ /dev/null @@ -1,131 +0,0 @@ -// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -import { Runloop } from '@runloop/api-client'; -import { Response } from 'node-fetch'; - -const client = new Runloop({ - bearerToken: 'My Bearer Token', - baseURL: process.env['TEST_API_BASE_URL'] ?? 'http://127.0.0.1:4010', -}); - -describe('resource runs', () => { - test('retrieve', async () => { - const responsePromise = client.benchmarks.runs.retrieve('id'); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('retrieve: request options instead of params are passed correctly', async () => { - // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error - await expect(client.benchmarks.runs.retrieve('id', { path: '/_stainless_unknown_path' })).rejects.toThrow( - Runloop.NotFoundError, - ); - }); - - test('list', async () => { - const responsePromise = client.benchmarks.runs.list(); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('list: request options instead of params are passed correctly', async () => { - // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error - await expect(client.benchmarks.runs.list({ path: '/_stainless_unknown_path' })).rejects.toThrow( - Runloop.NotFoundError, - ); - }); - - test('list: request options and params are passed correctly', async () => { - // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error - await expect( - client.benchmarks.runs.list( - { - benchmark_id: 'benchmark_id', - limit: 0, - name: 'name', - starting_after: 'starting_after', - }, - { path: '/_stainless_unknown_path' }, - ), - ).rejects.toThrow(Runloop.NotFoundError); - }); - - test('cancel', async () => { - const responsePromise = client.benchmarks.runs.cancel('id'); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('cancel: request options instead of params are passed correctly', async () => { - // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error - await expect(client.benchmarks.runs.cancel('id', { path: '/_stainless_unknown_path' })).rejects.toThrow( - Runloop.NotFoundError, - ); - }); - - test('complete', async () => { - const responsePromise = client.benchmarks.runs.complete('id'); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('complete: request options instead of params are passed correctly', async () => { - // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error - await expect(client.benchmarks.runs.complete('id', { path: '/_stainless_unknown_path' })).rejects.toThrow( - Runloop.NotFoundError, - ); - }); - - test('listScenarioRuns', async () => { - const responsePromise = client.benchmarks.runs.listScenarioRuns('id'); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('listScenarioRuns: request options instead of params are passed correctly', async () => { - // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error - await expect( - client.benchmarks.runs.listScenarioRuns('id', { path: '/_stainless_unknown_path' }), - ).rejects.toThrow(Runloop.NotFoundError); - }); - - test('listScenarioRuns: request options and params are passed correctly', async () => { - // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error - await expect( - client.benchmarks.runs.listScenarioRuns( - 'id', - { - limit: 0, - starting_after: 'starting_after', - state: 'running', - }, - { path: '/_stainless_unknown_path' }, - ), - ).rejects.toThrow(Runloop.NotFoundError); - }); -}); From adacd5383f6767c6e49571192d9ab64ed734acf7 Mon Sep 17 00:00:00 2001 From: Alexander Dines Date: Wed, 21 Jan 2026 17:08:42 -0800 Subject: [PATCH 6/7] cp dines --- src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index a4a1a60ed..328c69d65 100644 --- a/src/types.ts +++ b/src/types.ts @@ -92,7 +92,7 @@ export type * from './resources/scenarios/scorers'; // Benchmark Types // ============================================================================= -export type * from './resources/benchmarks/benchmarks'; +export type * from './resources/benchmarks'; // ============================================================================= // SDK Types From 82e7402759c042c68e31184c00e971df1bff2ac8 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 22 Jan 2026 01:09:04 +0000 Subject: [PATCH 7/7] release: 1.3.0 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 22 ++++++++++++++++++++++ package.json | 2 +- src/version.ts | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index d30af2c49..96f1cd949 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.2.0-beta-2" + ".": "1.3.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 04aeb718e..725d9c932 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 1.3.0 (2026-01-22) + +Full Changelog: [v1.2.0-beta-2...v1.3.0](https://github.com/runloopai/api-client-ts/compare/v1.2.0-beta-2...v1.3.0) + +### Features + +* adding network policies oo concepts ([#680](https://github.com/runloopai/api-client-ts/issues/680)) ([4f01339](https://github.com/runloopai/api-client-ts/commit/4f0133956634fefbc761de1183b6d900f7b2e28c)) +* **sdk:** Scenario ([#684](https://github.com/runloopai/api-client-ts/issues/684)) ([c7c3274](https://github.com/runloopai/api-client-ts/commit/c7c3274ac0dd74000414299c490346e71bcd1bf4)) +* **sdk:** ScenarioRun class ([#681](https://github.com/runloopai/api-client-ts/issues/681)) ([52a9275](https://github.com/runloopai/api-client-ts/commit/52a92754f31211dbc970f621ea931a05037de114)) + + +### Chores + +* fix stainless spec / remove deprecated benchmark/runs ([#7074](https://github.com/runloopai/api-client-ts/issues/7074)) ([d772c92](https://github.com/runloopai/api-client-ts/commit/d772c92acae9b124a797d204a0d1a9bdaa0eec19)) +* make smoketest workflow support workflow_call event ([#686](https://github.com/runloopai/api-client-ts/issues/686)) ([505de6b](https://github.com/runloopai/api-client-ts/commit/505de6b3b73fb355f606088cfcb4265fd3b25db5)) +* update network policy create params descriptions ([#7069](https://github.com/runloopai/api-client-ts/issues/7069)) ([159b43f](https://github.com/runloopai/api-client-ts/commit/159b43f2864105b83fa47cef591a216aedeb2b7c)) + + +### Refactors + +* **benchmark:** Deprecate /benchmark/{id}/runs in favor of /benchmark_runs ([#7019](https://github.com/runloopai/api-client-ts/issues/7019)) ([33624ce](https://github.com/runloopai/api-client-ts/commit/33624ce149003dfb1eaa49c10c8efa233b875ab9)) + ## 1.2.0-beta-2 (2026-01-20) Full Changelog: [v1.2.0-beta...v1.2.0-beta-2](https://github.com/runloopai/api-client-ts/compare/v1.2.0-beta...v1.2.0-beta-2) diff --git a/package.json b/package.json index f7e91ad5f..3dcd70fd3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@runloop/api-client", - "version": "1.2.0-beta-2", + "version": "1.3.0", "description": "The official TypeScript library for the Runloop API", "author": "Runloop ", "types": "dist/sdk.d.ts", diff --git a/src/version.ts b/src/version.ts index 7f028352e..b4178bdcc 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const VERSION = '1.2.0-beta-2'; // x-release-please-version +export const VERSION = '1.3.0-2'; // x-release-please-version