diff --git a/src/resources/scenarios/index.ts b/src/resources/scenarios/index.ts index bd2b01cc8..ae4589143 100644 --- a/src/resources/scenarios/index.ts +++ b/src/resources/scenarios/index.ts @@ -32,9 +32,7 @@ export { type ScorerRetrieveResponse, type ScorerUpdateResponse, type ScorerListResponse, - type ScorerValidateResponse, type ScorerCreateParams, type ScorerUpdateParams, type ScorerListParams, - type ScorerValidateParams, } from './scorers'; diff --git a/src/resources/scenarios/scenarios.ts b/src/resources/scenarios/scenarios.ts index 90d67878c..2c6c1275f 100644 --- a/src/resources/scenarios/scenarios.ts +++ b/src/resources/scenarios/scenarios.ts @@ -16,8 +16,6 @@ import { ScorerRetrieveResponse, ScorerUpdateParams, ScorerUpdateResponse, - ScorerValidateParams, - ScorerValidateResponse, Scorers, } from './scorers'; import { @@ -499,7 +497,6 @@ export interface ScoringFunction { | ScoringFunction.AstGrepScoringFunction | ScoringFunction.BashScriptScoringFunction | ScoringFunction.CommandScoringFunction - | ScoringFunction.CustomScoringFunction | ScoringFunction.PythonScriptScoringFunction | ScoringFunction.TestBasedScoringFunction; @@ -564,23 +561,6 @@ export namespace ScoringFunction { command?: string; } - /** - * CustomScoringFunction is a custom, user defined scoring function. - */ - export interface CustomScoringFunction { - /** - * Type of the scoring function, previously registered with Runloop. - */ - custom_scorer_type: string; - - type: 'custom_scorer'; - - /** - * Additional JSON structured context to pass to the scoring function. - */ - scorer_params?: unknown | null; - } - /** * PythonScriptScoringFunction will run a python script in the context of your * environment as a ScoringFunction. @@ -882,11 +862,9 @@ export declare namespace Scenarios { type ScorerRetrieveResponse as ScorerRetrieveResponse, type ScorerUpdateResponse as ScorerUpdateResponse, type ScorerListResponse as ScorerListResponse, - type ScorerValidateResponse as ScorerValidateResponse, ScorerListResponsesScenarioScorersCursorIDPage as ScorerListResponsesScenarioScorersCursorIDPage, type ScorerCreateParams as ScorerCreateParams, type ScorerUpdateParams as ScorerUpdateParams, type ScorerListParams as ScorerListParams, - type ScorerValidateParams as ScorerValidateParams, }; } diff --git a/src/resources/scenarios/scorers.ts b/src/resources/scenarios/scorers.ts index 5e5d0f892..86fbac8d8 100644 --- a/src/resources/scenarios/scorers.ts +++ b/src/resources/scenarios/scorers.ts @@ -3,7 +3,6 @@ import { APIResource } from '../../resource'; import { isRequestOptions } from '../../core'; import * as Core from '../../core'; -import * as ScenariosAPI from './scenarios'; import { ScenarioScorersCursorIDPage, type ScenarioScorersCursorIDPageParams } from '../../pagination'; export class Scorers extends APIResource { @@ -54,17 +53,6 @@ export class Scorers extends APIResource { ...options, }); } - - /** - * Validate a scenario scorer. - */ - validate( - id: string, - body: ScorerValidateParams, - options?: Core.RequestOptions, - ): Core.APIPromise { - return this._client.post(`/v1/scenarios/scorers/${id}/validate`, { body, ...options }); - } } export class ScorerListResponsesScenarioScorersCursorIDPage extends ScenarioScorersCursorIDPage {} @@ -149,28 +137,6 @@ export interface ScorerListResponse { type: string; } -export interface ScorerValidateResponse { - /** - * Name of the custom scorer. - */ - name: string; - - /** - * Json context that gets passed to the custom scorer - */ - scoring_context: unknown; - - /** - * Result of the scoring function. - */ - scoring_result: ScenariosAPI.ScoringFunctionResultView; - - /** - * The Environment in which the Scenario will run. - */ - environment_parameters?: ScenariosAPI.ScenarioEnvironment; -} - export interface ScorerCreateParams { /** * Bash script for the custom scorer taking context as a json object @@ -199,18 +165,6 @@ export interface ScorerUpdateParams { export interface ScorerListParams extends ScenarioScorersCursorIDPageParams {} -export interface ScorerValidateParams { - /** - * Json context that gets passed to the custom scorer - */ - scoring_context: unknown; - - /** - * The Environment in which the Scenario will run. - */ - environment_parameters?: ScenariosAPI.ScenarioEnvironment; -} - Scorers.ScorerListResponsesScenarioScorersCursorIDPage = ScorerListResponsesScenarioScorersCursorIDPage; export declare namespace Scorers { @@ -219,11 +173,9 @@ export declare namespace Scorers { type ScorerRetrieveResponse as ScorerRetrieveResponse, type ScorerUpdateResponse as ScorerUpdateResponse, type ScorerListResponse as ScorerListResponse, - type ScorerValidateResponse as ScorerValidateResponse, ScorerListResponsesScenarioScorersCursorIDPage as ScorerListResponsesScenarioScorersCursorIDPage, type ScorerCreateParams as ScorerCreateParams, type ScorerUpdateParams as ScorerUpdateParams, type ScorerListParams as ScorerListParams, - type ScorerValidateParams as ScorerValidateParams, }; } diff --git a/src/sdk.ts b/src/sdk.ts index 68cfc8d1e..19722dbfd 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -1303,14 +1303,14 @@ export class AgentOps { } /** - * Scorer SDK interface for managing custom scorers. + * Scorer SDK interface for managing scorers. * * @category Scorer * * @remarks * ## Overview * - * Scorers are custom scoring functions used to evaluate scenario outputs. A scorer is a + * Scorers are scoring functions used to evaluate scenario outputs. A scorer is a * script that runs and prints a score in the range [0.0, 1.0], e.g. `echo "0.5"`. * * ## Usage @@ -1331,22 +1331,9 @@ export class AgentOps { * }); * * // Update the scorer - * await scorer.update({ bash_script: 'echo "0.5"' }); - * - * // Validate the scorer with a scoring context - * const result = await scorer.validate({ scoring_context: { output: 'hello' } }); - * console.log(result.scoring_result.score); - * ``` - * - * @example - * Get scorer info (typical usage): - * ```typescript - * const runloop = new RunloopSDK(); - * const scorer = await runloop.scorer.create({ - * type: 'my_scorer', - * bash_script: 'echo "1.0"', - * }); + * await scorer.update({ type: 'my_scorer', bash_script: 'echo "0.5"' }); * + * // Get scorer info * const info = await scorer.getInfo(); * console.log(`Scorer ${info.id} (${info.type})`); * ``` diff --git a/src/sdk/scorer.ts b/src/sdk/scorer.ts index 00241fca0..3d9129f3b 100644 --- a/src/sdk/scorer.ts +++ b/src/sdk/scorer.ts @@ -6,19 +6,17 @@ import type { ScorerRetrieveResponse, ScorerUpdateResponse, ScorerUpdateParams, - ScorerValidateParams, - ScorerValidateResponse, } from '../resources/scenarios/scorers'; /** - * Object-oriented interface for working with custom Scorers. + * Object-oriented interface for working with Scorers. * * @category Scorer * * @remarks * ## Overview * - * The `Scorer` class provides a high-level, object-oriented API for managing custom scorers. + * The `Scorer` class provides a high-level, object-oriented API for managing scorers. * Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs. * * ## Quickstart @@ -32,8 +30,8 @@ import type { * bash_script: 'echo "1.0"', * }); * - * const result = await scorer.validate({ scoring_context: { output: 'test' } }); - * console.log(`Score: ${result.scoring_result.score}`); + * const info = await scorer.getInfo(); + * console.log(`Scorer type: ${info.type}`); * ``` */ export class Scorer { @@ -176,27 +174,4 @@ export class Scorer { async update(params: ScorerUpdateParams, options?: Core.RequestOptions): Promise { return this.client.scenarios.scorers.update(this._id, params, options); } - - /** - * Run the scorer against the provided context and return the result. - * - * @example - * ```typescript - * const result = await scorer.validate({ - * scoring_context: { output: 'test output', expected: 'test output' } - * }); - * console.log(`Validation score: ${result.scoring_result.score}`); - * console.log(`Output: ${result.scoring_result.output}`); - * ``` - * - * @param {ScorerValidateParams} params - Validation parameters - * @param {Core.RequestOptions} [options] - Request options - * @returns {Promise} Validation result with score - */ - async validate( - params: ScorerValidateParams, - options?: Core.RequestOptions, - ): Promise { - return this.client.scenarios.scorers.validate(this._id, params, options); - } } diff --git a/tests/api-resources/scenarios/scorers.test.ts b/tests/api-resources/scenarios/scorers.test.ts index a5c36efe7..b78ab2bb0 100644 --- a/tests/api-resources/scenarios/scorers.test.ts +++ b/tests/api-resources/scenarios/scorers.test.ts @@ -90,40 +90,4 @@ describe('resource scorers', () => { ), ).rejects.toThrow(Runloop.NotFoundError); }); - - test('validate: only required params', async () => { - const responsePromise = client.scenarios.scorers.validate('id', { scoring_context: {} }); - const rawResponse = await responsePromise.asResponse(); - expect(rawResponse).toBeInstanceOf(Response); - const response = await responsePromise; - expect(response).not.toBeInstanceOf(Response); - const dataAndResponse = await responsePromise.withResponse(); - expect(dataAndResponse.data).toBe(response); - expect(dataAndResponse.response).toBe(rawResponse); - }); - - test('validate: required and optional params', async () => { - const response = await client.scenarios.scorers.validate('id', { - scoring_context: {}, - environment_parameters: { - blueprint_id: 'blueprint_id', - launch_parameters: { - after_idle: { idle_time_seconds: 0, on_idle: 'shutdown' }, - architecture: 'x86_64', - available_ports: [0], - custom_cpu_cores: 0, - custom_disk_size: 0, - custom_gb_memory: 0, - keep_alive_time_seconds: 0, - launch_commands: ['string'], - network_policy_id: 'network_policy_id', - required_services: ['string'], - resource_size_request: 'X_SMALL', - user_parameters: { uid: 0, username: 'username' }, - }, - snapshot_id: 'snapshot_id', - working_directory: 'working_directory', - }, - }); - }); }); diff --git a/tests/objects/scorer.test.ts b/tests/objects/scorer.test.ts index 337555a1b..fbe899a7c 100644 --- a/tests/objects/scorer.test.ts +++ b/tests/objects/scorer.test.ts @@ -1,9 +1,5 @@ import { Scorer } from '../../src/sdk/scorer'; -import type { - ScorerRetrieveResponse, - ScorerUpdateResponse, - ScorerValidateResponse, -} from '../../src/resources/scenarios/scorers'; +import type { ScorerRetrieveResponse, ScorerUpdateResponse } from '../../src/resources/scenarios/scorers'; // Mock the Runloop client jest.mock('../../src/index'); @@ -11,7 +7,6 @@ jest.mock('../../src/index'); describe('Scorer', () => { let mockClient: any; let mockScorerData: ScorerRetrieveResponse; - let mockValidateResult: ScorerValidateResponse; beforeEach(() => { mockClient = { @@ -20,7 +15,6 @@ describe('Scorer', () => { create: jest.fn(), retrieve: jest.fn(), update: jest.fn(), - validate: jest.fn(), list: jest.fn(), }, }, @@ -31,17 +25,6 @@ describe('Scorer', () => { type: 'my_custom_scorer', bash_script: 'echo "1.0"', }; - - mockValidateResult = { - name: 'my_custom_scorer', - scoring_context: { output: 'test' }, - scoring_result: { - output: '1.0', - score: 1.0, - state: 'complete', - scoring_function_name: 'test-scorer', - }, - }; }); describe('fromId', () => { @@ -188,38 +171,6 @@ describe('Scorer', () => { }); }); - describe('validate', () => { - it('should validate scorer with scoring context', async () => { - mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); - - const scorer = Scorer.fromId(mockClient, 'scs_123'); - const result = await scorer.validate({ - scoring_context: { output: 'test output', expected: 'expected output' }, - }); - - expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( - 'scs_123', - { scoring_context: { output: 'test output', expected: 'expected output' } }, - undefined, - ); - expect(result.scoring_result.score).toBe(1.0); - expect(result.scoring_result.output).toBe('1.0'); - }); - - it('should pass options to the API client', async () => { - mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); - - const scorer = Scorer.fromId(mockClient, 'scs_123'); - await scorer.validate({ scoring_context: {} }, { timeout: 30000 }); - - expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( - 'scs_123', - { scoring_context: {} }, - { timeout: 30000 }, - ); - }); - }); - describe('error handling', () => { it('should handle API errors on getInfo', async () => { const error = new Error('Scorer not found'); @@ -240,14 +191,5 @@ describe('Scorer', () => { 'Update failed', ); }); - - it('should handle API errors on validate', async () => { - const error = new Error('Validation timeout'); - mockClient.scenarios.scorers.validate.mockRejectedValue(error); - - const scorer = Scorer.fromId(mockClient, 'scs_123'); - - await expect(scorer.validate({ scoring_context: {} })).rejects.toThrow('Validation timeout'); - }); }); });