chore(benchmarks): removed custom scorers

src/resources/scenarios/index.ts

@@ -32,9 +32,7 @@ export {
   type ScorerRetrieveResponse,
   type ScorerUpdateResponse,
   type ScorerListResponse,
-  type ScorerValidateResponse,
   type ScorerCreateParams,
   type ScorerUpdateParams,
   type ScorerListParams,
-  type ScorerValidateParams,
 } from './scorers';

src/resources/scenarios/scenarios.ts

@@ -16,8 +16,6 @@ import {
   ScorerRetrieveResponse,
   ScorerUpdateParams,
   ScorerUpdateResponse,
-  ScorerValidateParams,
-  ScorerValidateResponse,
   Scorers,
 } from './scorers';
 import {
@@ -499,7 +497,6 @@ export interface ScoringFunction {
     | ScoringFunction.AstGrepScoringFunction
     | ScoringFunction.BashScriptScoringFunction
     | ScoringFunction.CommandScoringFunction
-    | ScoringFunction.CustomScoringFunction
     | ScoringFunction.PythonScriptScoringFunction
     | ScoringFunction.TestBasedScoringFunction;
 
@@ -564,23 +561,6 @@ export namespace ScoringFunction {
     command?: string;
   }
 
-  /**
-   * CustomScoringFunction is a custom, user defined scoring function.
-   */
-  export interface CustomScoringFunction {
-    /**
-     * Type of the scoring function, previously registered with Runloop.
-     */
-    custom_scorer_type: string;
-
-    type: 'custom_scorer';
-
-    /**
-     * Additional JSON structured context to pass to the scoring function.
-     */
-    scorer_params?: unknown | null;
-  }
-
   /**
    * PythonScriptScoringFunction will run a python script in the context of your
    * environment as a ScoringFunction.
@@ -882,11 +862,9 @@ export declare namespace Scenarios {
     type ScorerRetrieveResponse as ScorerRetrieveResponse,
     type ScorerUpdateResponse as ScorerUpdateResponse,
     type ScorerListResponse as ScorerListResponse,
-    type ScorerValidateResponse as ScorerValidateResponse,
     ScorerListResponsesScenarioScorersCursorIDPage as ScorerListResponsesScenarioScorersCursorIDPage,
     type ScorerCreateParams as ScorerCreateParams,
     type ScorerUpdateParams as ScorerUpdateParams,
     type ScorerListParams as ScorerListParams,
-    type ScorerValidateParams as ScorerValidateParams,
   };
 }

src/resources/scenarios/scorers.ts

@@ -3,7 +3,6 @@
 import { APIResource } from '../../resource';
 import { isRequestOptions } from '../../core';
 import * as Core from '../../core';
-import * as ScenariosAPI from './scenarios';
 import { ScenarioScorersCursorIDPage, type ScenarioScorersCursorIDPageParams } from '../../pagination';
 
 export class Scorers extends APIResource {
@@ -54,17 +53,6 @@ export class Scorers extends APIResource {
       ...options,
     });
   }
-
-  /**
-   * Validate a scenario scorer.
-   */
-  validate(
-    id: string,
-    body: ScorerValidateParams,
-    options?: Core.RequestOptions,
-  ): Core.APIPromise<ScorerValidateResponse> {
-    return this._client.post(`/v1/scenarios/scorers/${id}/validate`, { body, ...options });
-  }
 }
 
 export class ScorerListResponsesScenarioScorersCursorIDPage extends ScenarioScorersCursorIDPage<ScorerListResponse> {}
@@ -149,28 +137,6 @@ export interface ScorerListResponse {
   type: string;
 }
 
-export interface ScorerValidateResponse {
-  /**
-   * Name of the custom scorer.
-   */
-  name: string;
-
-  /**
-   * Json context that gets passed to the custom scorer
-   */
-  scoring_context: unknown;
-
-  /**
-   * Result of the scoring function.
-   */
-  scoring_result: ScenariosAPI.ScoringFunctionResultView;
-
-  /**
-   * The Environment in which the Scenario will run.
-   */
-  environment_parameters?: ScenariosAPI.ScenarioEnvironment;
-}
-
 export interface ScorerCreateParams {
   /**
    * Bash script for the custom scorer taking context as a json object
@@ -199,18 +165,6 @@ export interface ScorerUpdateParams {
 
 export interface ScorerListParams extends ScenarioScorersCursorIDPageParams {}
 
-export interface ScorerValidateParams {
-  /**
-   * Json context that gets passed to the custom scorer
-   */
-  scoring_context: unknown;
-
-  /**
-   * The Environment in which the Scenario will run.
-   */
-  environment_parameters?: ScenariosAPI.ScenarioEnvironment;
-}
-
 Scorers.ScorerListResponsesScenarioScorersCursorIDPage = ScorerListResponsesScenarioScorersCursorIDPage;
 
 export declare namespace Scorers {
@@ -219,11 +173,9 @@ export declare namespace Scorers {
     type ScorerRetrieveResponse as ScorerRetrieveResponse,
     type ScorerUpdateResponse as ScorerUpdateResponse,
     type ScorerListResponse as ScorerListResponse,
-    type ScorerValidateResponse as ScorerValidateResponse,
     ScorerListResponsesScenarioScorersCursorIDPage as ScorerListResponsesScenarioScorersCursorIDPage,
     type ScorerCreateParams as ScorerCreateParams,
     type ScorerUpdateParams as ScorerUpdateParams,
     type ScorerListParams as ScorerListParams,
-    type ScorerValidateParams as ScorerValidateParams,
   };
 }

src/sdk.ts

@@ -1303,14 +1303,14 @@ export class AgentOps {
 }
 
 /**
- * Scorer SDK interface for managing custom scorers.
+ * Scorer SDK interface for managing scorers.
  *
  * @category Scorer
  *
  * @remarks
  * ## Overview
  *
- * Scorers are custom scoring functions used to evaluate scenario outputs. A scorer is a
+ * Scorers are scoring functions used to evaluate scenario outputs. A scorer is a
  * script that runs and prints a score in the range [0.0, 1.0], e.g. `echo "0.5"`.
  *
  * ## Usage
@@ -1331,22 +1331,9 @@ export class AgentOps {
  * });
  *
  * // Update the scorer
- * await scorer.update({ bash_script: 'echo "0.5"' });
- *
- * // Validate the scorer with a scoring context
- * const result = await scorer.validate({ scoring_context: { output: 'hello' } });
- * console.log(result.scoring_result.score);
- * ```
- *
- * @example
- * Get scorer info (typical usage):
- * ```typescript
- * const runloop = new RunloopSDK();
- * const scorer = await runloop.scorer.create({
- *   type: 'my_scorer',
- *   bash_script: 'echo "1.0"',
- * });
+ * await scorer.update({ type: 'my_scorer', bash_script: 'echo "0.5"' });
  *
+ * // Get scorer info
  * const info = await scorer.getInfo();
  * console.log(`Scorer ${info.id} (${info.type})`);
  * ```

src/sdk/scorer.ts

@@ -6,19 +6,17 @@ import type {
   ScorerRetrieveResponse,
   ScorerUpdateResponse,
   ScorerUpdateParams,
-  ScorerValidateParams,
-  ScorerValidateResponse,
 } from '../resources/scenarios/scorers';
 
 /**
- * Object-oriented interface for working with custom Scorers.
+ * Object-oriented interface for working with Scorers.
  *
  * @category Scorer
  *
  * @remarks
  * ## Overview
  *
- * The `Scorer` class provides a high-level, object-oriented API for managing custom scorers.
+ * The `Scorer` class provides a high-level, object-oriented API for managing scorers.
  * Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs.
  *
  * ## Quickstart
@@ -32,8 +30,8 @@ import type {
  *   bash_script: 'echo "1.0"',
  * });
  *
- * const result = await scorer.validate({ scoring_context: { output: 'test' } });
- * console.log(`Score: ${result.scoring_result.score}`);
+ * const info = await scorer.getInfo();
+ * console.log(`Scorer type: ${info.type}`);
  * ```
  */
 export class Scorer {
@@ -176,27 +174,4 @@ export class Scorer {
   async update(params: ScorerUpdateParams, options?: Core.RequestOptions): Promise<ScorerUpdateResponse> {
     return this.client.scenarios.scorers.update(this._id, params, options);
   }
-
-  /**
-   * Run the scorer against the provided context and return the result.
-   *
-   * @example
-   * ```typescript
-   * const result = await scorer.validate({
-   *   scoring_context: { output: 'test output', expected: 'test output' }
-   * });
-   * console.log(`Validation score: ${result.scoring_result.score}`);
-   * console.log(`Output: ${result.scoring_result.output}`);
-   * ```
-   *
-   * @param {ScorerValidateParams} params - Validation parameters
-   * @param {Core.RequestOptions} [options] - Request options
-   * @returns {Promise<ScorerValidateResponse>} Validation result with score
-   */
-  async validate(
-    params: ScorerValidateParams,
-    options?: Core.RequestOptions,
-  ): Promise<ScorerValidateResponse> {
-    return this.client.scenarios.scorers.validate(this._id, params, options);
-  }
 }

tests/api-resources/scenarios/scorers.test.ts

@@ -90,40 +90,4 @@ describe('resource scorers', () => {
       ),
     ).rejects.toThrow(Runloop.NotFoundError);
   });
-
-  test('validate: only required params', async () => {
-    const responsePromise = client.scenarios.scorers.validate('id', { scoring_context: {} });
-    const rawResponse = await responsePromise.asResponse();
-    expect(rawResponse).toBeInstanceOf(Response);
-    const response = await responsePromise;
-    expect(response).not.toBeInstanceOf(Response);
-    const dataAndResponse = await responsePromise.withResponse();
-    expect(dataAndResponse.data).toBe(response);
-    expect(dataAndResponse.response).toBe(rawResponse);
-  });
-
-  test('validate: required and optional params', async () => {
-    const response = await client.scenarios.scorers.validate('id', {
-      scoring_context: {},
-      environment_parameters: {
-        blueprint_id: 'blueprint_id',
-        launch_parameters: {
-          after_idle: { idle_time_seconds: 0, on_idle: 'shutdown' },
-          architecture: 'x86_64',
-          available_ports: [0],
-          custom_cpu_cores: 0,
-          custom_disk_size: 0,
-          custom_gb_memory: 0,
-          keep_alive_time_seconds: 0,
-          launch_commands: ['string'],
-          network_policy_id: 'network_policy_id',
-          required_services: ['string'],
-          resource_size_request: 'X_SMALL',
-          user_parameters: { uid: 0, username: 'username' },
-        },
-        snapshot_id: 'snapshot_id',
-        working_directory: 'working_directory',
-      },
-    });
-  });
 });

tests/objects/scorer.test.ts

@@ -1,17 +1,12 @@
 import { Scorer } from '../../src/sdk/scorer';
-import type {
-  ScorerRetrieveResponse,
-  ScorerUpdateResponse,
-  ScorerValidateResponse,
-} from '../../src/resources/scenarios/scorers';
+import type { ScorerRetrieveResponse, ScorerUpdateResponse } from '../../src/resources/scenarios/scorers';
 
 // Mock the Runloop client
 jest.mock('../../src/index');
 
 describe('Scorer', () => {
   let mockClient: any;
   let mockScorerData: ScorerRetrieveResponse;
-  let mockValidateResult: ScorerValidateResponse;
 
   beforeEach(() => {
     mockClient = {
@@ -20,7 +15,6 @@ describe('Scorer', () => {
           create: jest.fn(),
           retrieve: jest.fn(),
           update: jest.fn(),
-          validate: jest.fn(),
           list: jest.fn(),
         },
       },
@@ -31,17 +25,6 @@ describe('Scorer', () => {
       type: 'my_custom_scorer',
       bash_script: 'echo "1.0"',
     };
-
-    mockValidateResult = {
-      name: 'my_custom_scorer',
-      scoring_context: { output: 'test' },
-      scoring_result: {
-        output: '1.0',
-        score: 1.0,
-        state: 'complete',
-        scoring_function_name: 'test-scorer',
-      },
-    };
   });
 
   describe('fromId', () => {
@@ -188,38 +171,6 @@ describe('Scorer', () => {
     });
   });
 
-  describe('validate', () => {
-    it('should validate scorer with scoring context', async () => {
-      mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult);
-
-      const scorer = Scorer.fromId(mockClient, 'scs_123');
-      const result = await scorer.validate({
-        scoring_context: { output: 'test output', expected: 'expected output' },
-      });
-
-      expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith(
-        'scs_123',
-        { scoring_context: { output: 'test output', expected: 'expected output' } },
-        undefined,
-      );
-      expect(result.scoring_result.score).toBe(1.0);
-      expect(result.scoring_result.output).toBe('1.0');
-    });
-
-    it('should pass options to the API client', async () => {
-      mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult);
-
-      const scorer = Scorer.fromId(mockClient, 'scs_123');
-      await scorer.validate({ scoring_context: {} }, { timeout: 30000 });
-
-      expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith(
-        'scs_123',
-        { scoring_context: {} },
-        { timeout: 30000 },
-      );
-    });
-  });
-
   describe('error handling', () => {
     it('should handle API errors on getInfo', async () => {
       const error = new Error('Scorer not found');
@@ -240,14 +191,5 @@ describe('Scorer', () => {
         'Update failed',
       );
     });
-
-    it('should handle API errors on validate', async () => {
-      const error = new Error('Validation timeout');
-      mockClient.scenarios.scorers.validate.mockRejectedValue(error);
-
-      const scorer = Scorer.fromId(mockClient, 'scs_123');
-
-      await expect(scorer.validate({ scoring_context: {} })).rejects.toThrow('Validation timeout');
-    });
   });
 });