From 1e6fe5683a9bbe74abc5f903f4c9d99d6b058d9a Mon Sep 17 00:00:00 2001 From: James Chainey Date: Thu, 18 Dec 2025 10:11:34 -0800 Subject: [PATCH 1/8] feat(sdk): added scorers to OO SDK --- src/sdk.ts | 67 +++++++ src/sdk/index.ts | 1 + src/sdk/scorer.ts | 125 ++++++++++++ src/types.ts | 1 + tests/objects/scorer.test.ts | 194 +++++++++++++++++++ tests/smoketests/object-oriented/sdk.test.ts | 1 + 6 files changed, 389 insertions(+) create mode 100644 src/sdk/scorer.ts create mode 100644 tests/objects/scorer.test.ts diff --git a/src/sdk.ts b/src/sdk.ts index 8bf069e59..2cf89563d 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -6,6 +6,7 @@ import { Blueprint, type CreateParams as BlueprintCreateParams } from './sdk/blu import { Snapshot } from './sdk/snapshot'; import { StorageObject } from './sdk/storage-object'; import { Agent } from './sdk/agent'; +import { Scorer } from './sdk/scorer'; // Import types used in this file import type { @@ -17,6 +18,7 @@ import type { import type { BlueprintListParams } from './resources/blueprints'; import type { ObjectCreateParams, ObjectListParams } from './resources/objects'; import type { AgentCreateParams, AgentListParams } from './resources/agents'; +import type { ScorerCreateParams, ScorerListParams } from './resources/scenarios/scorers'; import { PollingOptions } from './lib/polling'; import * as Shared from './resources/shared'; @@ -190,6 +192,7 @@ type ContentType = ObjectCreateParams['content_type']; * - `snapshot` - {@link SnapshotOps} * - `storageObject` - {@link StorageObjectOps} * - `agent` - {@link AgentOps} + * - `scorer` - {@link ScorerOps} * * See the documentation for each Operations class for more details. * @@ -260,6 +263,14 @@ export class RunloopSDK { */ public readonly agent: AgentOps; + /** + * **Scorer Operations** - {@link ScorerOps} for creating and accessing {@link Scorer} class instances. + * + * Scorers are custom scoring functions that evaluate scenario outputs. They define scripts + * that produce a score in the range [0.0, 1.0] for scenario runs. + */ + public readonly scorer: ScorerOps; + /** * Creates a new RunloopSDK instance. * @param {ClientOptions} [options] - Optional client configuration options. @@ -271,6 +282,7 @@ export class RunloopSDK { this.snapshot = new SnapshotOps(this.api); this.storageObject = new StorageObjectOps(this.api); this.agent = new AgentOps(this.api); + this.scorer = new ScorerOps(this.api); } } @@ -1255,6 +1267,58 @@ export class AgentOps { } } +/** + * Scorer SDK interface for managing custom scorers. + * + * @category Scorer + */ +export class ScorerOps { + /** + * @private + */ + constructor(private client: RunloopAPI) {} + + /** + * Create a new custom scorer. + * + * @param {ScorerCreateParams} params - Parameters for creating the scorer + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} A {@link Scorer} instance + */ + async create(params: ScorerCreateParams, options?: Core.RequestOptions): Promise { + const response = await this.client.scenarios.scorers.create(params, options); + return Scorer.fromId(this.client, response.id); + } + + /** + * Get a scorer object by its ID. + * + * @param {string} id - The ID of the scorer + * @returns {Scorer} A {@link Scorer} instance + */ + fromId(id: string): Scorer { + return Scorer.fromId(this.client, id); + } + + /** + * List all scorers with optional filters. + * + * @param {ScorerListParams} [params] - Optional filter parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} An array of {@link Scorer} instances + */ + async list(params?: ScorerListParams, options?: Core.RequestOptions): Promise { + const page = await this.client.scenarios.scorers.list(params, options); + const scorers: Scorer[] = []; + + for await (const scorer of page) { + scorers.push(Scorer.fromId(this.client, scorer.id)); + } + + return scorers; + } +} + // @deprecated Use {@link RunloopSDK} instead. /** * @deprecated Use {@link RunloopSDK} instead. @@ -1275,11 +1339,13 @@ export declare namespace RunloopSDK { SnapshotOps as SnapshotOps, StorageObjectOps as StorageObjectOps, AgentOps as AgentOps, + ScorerOps as ScorerOps, Devbox as Devbox, Blueprint as Blueprint, Snapshot as Snapshot, StorageObject as StorageObject, Agent as Agent, + Scorer as Scorer, }; } // Export SDK classes from sdk/sdk.ts - these are separate from RunloopSDK to avoid circular dependencies @@ -1293,6 +1359,7 @@ export { Snapshot, StorageObject, Agent, + Scorer, Execution, ExecutionResult, } from './sdk/index'; diff --git a/src/sdk/index.ts b/src/sdk/index.ts index 65ce3bf1d..38cb0ce25 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -5,3 +5,4 @@ export { StorageObject } from './storage-object'; export { Agent } from './agent'; export { Execution } from './execution'; export { ExecutionResult } from './execution-result'; +export { Scorer } from './scorer'; diff --git a/src/sdk/scorer.ts b/src/sdk/scorer.ts new file mode 100644 index 000000000..f52b29e0a --- /dev/null +++ b/src/sdk/scorer.ts @@ -0,0 +1,125 @@ +import { Runloop } from '../index'; +import type * as Core from '../core'; +import type { + ScorerRetrieveResponse, + ScorerUpdateResponse, + ScorerUpdateParams, + ScorerValidateParams, + ScorerValidateResponse, +} from '../resources/scenarios/scorers'; + +/** + * Object-oriented interface for working with custom Scorers. + * + * @category Scorer + * + * @remarks + * ## Overview + * + * The `Scorer` class provides a high-level API for managing custom scorers. + * Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs. + * + * ## Usage + * + * Obtain instances via `runloop.scorer.create()` or `runloop.scorer.fromId()`: + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "score=1.0"' + * }); + * + * // Validate the scorer + * const result = await scorer.validate({ scoring_context: { output: 'test' } }); + * console.log(`Score: ${result.scoring_result.score}`); + * ``` + */ +export class Scorer { + private client: Runloop; + private _id: string; + + private constructor(client: Runloop, scorerId: string) { + this.client = client; + this._id = scorerId; + } + + /** + * Create a Scorer instance by ID without retrieving from API. + * Use getInfo() to fetch the actual data when needed. + * + * @param {Runloop} client - The Runloop client instance + * @param {string} id - The scorer ID + * @returns {Scorer} A Scorer instance + */ + static fromId(client: Runloop, id: string): Scorer { + return new Scorer(client, id); + } + + /** + * Get the scorer ID. + * @returns {string} The scorer ID + */ + get id(): string { + return this._id; + } + + /** + * Fetch current scorer details from the API. + * + * @example + * ```typescript + * const info = await scorer.getInfo(); + * console.log(`Scorer type: ${info.type}`); + * ``` + * + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Current scorer details + */ + async getInfo(options?: Core.RequestOptions): Promise { + return this.client.scenarios.scorers.retrieve(this._id, options); + } + + /** + * Update the scorer's type or bash script. + * + * @example + * ```typescript + * const updated = await scorer.update({ + * type: 'my_scorer_v2', + * bash_script: 'echo "score=0.5"' + * }); + * ``` + * + * @param {ScorerUpdateParams} params - Update parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Updated scorer details + */ + async update(params: ScorerUpdateParams, options?: Core.RequestOptions): Promise { + return this.client.scenarios.scorers.update(this._id, params, options); + } + + /** + * Run the scorer against the provided context and return the result. + * + * @example + * ```typescript + * const result = await scorer.validate({ + * scoring_context: { output: 'test output', expected: 'test output' } + * }); + * console.log(`Validation score: ${result.scoring_result.score}`); + * console.log(`Output: ${result.scoring_result.output}`); + * ``` + * + * @param {ScorerValidateParams} params - Validation parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Validation result with score + */ + async validate( + params: ScorerValidateParams, + options?: Core.RequestOptions, + ): Promise { + return this.client.scenarios.scorers.validate(this._id, params, options); + } +} diff --git a/src/types.ts b/src/types.ts index c4f12af36..a4a1a60ed 100644 --- a/src/types.ts +++ b/src/types.ts @@ -86,6 +86,7 @@ export type * from './resources/secrets'; // ============================================================================= export type * from './resources/scenarios/scenarios'; +export type * from './resources/scenarios/scorers'; // ============================================================================= // Benchmark Types diff --git a/tests/objects/scorer.test.ts b/tests/objects/scorer.test.ts new file mode 100644 index 000000000..a6ec80bb7 --- /dev/null +++ b/tests/objects/scorer.test.ts @@ -0,0 +1,194 @@ +import { Scorer } from '../../src/sdk/scorer'; +import type { + ScorerRetrieveResponse, + ScorerUpdateResponse, + ScorerValidateResponse, +} from '../../src/resources/scenarios/scorers'; + +// Mock the Runloop client +jest.mock('../../src/index'); + +describe('Scorer', () => { + let mockClient: any; + let mockScorerData: ScorerRetrieveResponse; + let mockValidateResult: ScorerValidateResponse; + + beforeEach(() => { + mockClient = { + scenarios: { + scorers: { + retrieve: jest.fn(), + update: jest.fn(), + validate: jest.fn(), + }, + }, + } as any; + + mockScorerData = { + id: 'scr-123', + type: 'my_custom_scorer', + bash_script: 'echo "score=1.0"', + }; + + mockValidateResult = { + name: 'my_custom_scorer', + scoring_context: { output: 'test' }, + scoring_result: { + output: 'score=1.0', + score: 1.0, + state: 'complete', + scoring_function_name: 'test-scorer', + }, + }; + }); + + describe('fromId', () => { + it('should create a Scorer instance by ID without API call', () => { + const scorer = Scorer.fromId(mockClient, 'scr-123'); + + expect(scorer).toBeInstanceOf(Scorer); + expect(scorer.id).toBe('scr-123'); + }); + }); + + describe('getInfo', () => { + it('should retrieve scorer information from API', async () => { + mockClient.scenarios.scorers.retrieve.mockResolvedValue(mockScorerData); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + const info = await scorer.getInfo(); + + expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('scr-123', undefined); + expect(info).toEqual(mockScorerData); + expect(info.type).toBe('my_custom_scorer'); + expect(info.bash_script).toBe('echo "score=1.0"'); + }); + + it('should pass options to the API client', async () => { + mockClient.scenarios.scorers.retrieve.mockResolvedValue(mockScorerData); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + await scorer.getInfo({ timeout: 5000 }); + + expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('scr-123', { timeout: 5000 }); + }); + }); + + describe('update', () => { + it('should update scorer with provided parameters', async () => { + const updatedData: ScorerUpdateResponse = { + ...mockScorerData, + type: 'my_custom_scorer_v2', + bash_script: 'echo "score=0.5"', + }; + mockClient.scenarios.scorers.update.mockResolvedValue(updatedData); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + const result = await scorer.update({ + type: 'my_custom_scorer_v2', + bash_script: 'echo "score=0.5"', + }); + + expect(mockClient.scenarios.scorers.update).toHaveBeenCalledWith( + 'scr-123', + { type: 'my_custom_scorer_v2', bash_script: 'echo "score=0.5"' }, + undefined, + ); + expect(result.type).toBe('my_custom_scorer_v2'); + }); + + it('should pass options to the API client', async () => { + mockClient.scenarios.scorers.update.mockResolvedValue(mockScorerData); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + await scorer.update({ type: 'updated', bash_script: 'echo "score=1.0"' }, { timeout: 5000 }); + + expect(mockClient.scenarios.scorers.update).toHaveBeenCalledWith( + 'scr-123', + { type: 'updated', bash_script: 'echo "score=1.0"' }, + { timeout: 5000 }, + ); + }); + }); + + describe('validate', () => { + it('should validate scorer with scoring context', async () => { + mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + const result = await scorer.validate({ + scoring_context: { output: 'test output', expected: 'expected output' }, + }); + + expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( + 'scr-123', + { scoring_context: { output: 'test output', expected: 'expected output' } }, + undefined, + ); + expect(result.scoring_result.score).toBe(1.0); + expect(result.scoring_result.output).toBe('score=1.0'); + }); + + it('should pass options to the API client', async () => { + mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + await scorer.validate({ scoring_context: {} }, { timeout: 30000 }); + + expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( + 'scr-123', + { scoring_context: {} }, + { timeout: 30000 }, + ); + }); + + it('should handle low score validation result', async () => { + const lowScoreResult: ScorerValidateResponse = { + name: 'my_custom_scorer', + scoring_context: { incorrect: true }, + scoring_result: { + output: 'score=0.25', + score: 0.25, + state: 'complete', + scoring_function_name: 'test-scorer', + }, + }; + mockClient.scenarios.scorers.validate.mockResolvedValue(lowScoreResult); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + const result = await scorer.validate({ scoring_context: { incorrect: true } }); + + expect(result.scoring_result.score).toBe(0.25); + }); + }); + + describe('error handling', () => { + it('should handle API errors on getInfo', async () => { + const error = new Error('Scorer not found'); + mockClient.scenarios.scorers.retrieve.mockRejectedValue(error); + + const scorer = Scorer.fromId(mockClient, 'scr-nonexistent'); + + await expect(scorer.getInfo()).rejects.toThrow('Scorer not found'); + }); + + it('should handle API errors on update', async () => { + const error = new Error('Update failed'); + mockClient.scenarios.scorers.update.mockRejectedValue(error); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + + await expect(scorer.update({ type: 'new_type', bash_script: 'echo 1' })).rejects.toThrow('Update failed'); + }); + + it('should handle API errors on validate', async () => { + const error = new Error('Validation timeout'); + mockClient.scenarios.scorers.validate.mockRejectedValue(error); + + const scorer = Scorer.fromId(mockClient, 'scr-123'); + + await expect(scorer.validate({ scoring_context: {} })).rejects.toThrow('Validation timeout'); + }); + }); +}); + diff --git a/tests/smoketests/object-oriented/sdk.test.ts b/tests/smoketests/object-oriented/sdk.test.ts index c7c5ce2c6..f086ff48c 100644 --- a/tests/smoketests/object-oriented/sdk.test.ts +++ b/tests/smoketests/object-oriented/sdk.test.ts @@ -10,6 +10,7 @@ describe('smoketest: object-oriented SDK', () => { expect(sdk.blueprint).toBeDefined(); expect(sdk.snapshot).toBeDefined(); expect(sdk.storageObject).toBeDefined(); + expect(sdk.scorer).toBeDefined(); expect(sdk.api).toBeDefined(); }); From f3e47a412bc85edd35ead25ad3170ab90600b2d9 Mon Sep 17 00:00:00 2001 From: James Chainey Date: Thu, 18 Dec 2025 13:40:36 -0800 Subject: [PATCH 2/8] added smoketest --- .../smoketests/object-oriented/scorer.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/smoketests/object-oriented/scorer.test.ts diff --git a/tests/smoketests/object-oriented/scorer.test.ts b/tests/smoketests/object-oriented/scorer.test.ts new file mode 100644 index 000000000..b5aeedfce --- /dev/null +++ b/tests/smoketests/object-oriented/scorer.test.ts @@ -0,0 +1,64 @@ +import { makeClientSDK, uniqueName } from '../utils'; +import { Scorer } from '@runloop/api-client/sdk'; + +const sdk = makeClientSDK(); + +describe('smoketest: object-oriented scorers', () => { + describe('scorer lifecycle', () => { + let scorer: Scorer; + let scorerId: string | undefined; + + beforeAll(async () => { + // Create a scorer first (no delete endpoint currently, so we keep it small + uniquely named) + scorer = await sdk.scorer.create({ + type: uniqueName('sdk-scorer'), + bash_script: 'echo "score=1.0"', + }); + expect(scorer).toBeDefined(); + expect(scorer.id).toBeTruthy(); + scorerId = scorer.id; + }); + + test('get scorer info', async () => { + const info = await scorer.getInfo(); + expect(info.id).toBe(scorerId); + expect(info.type).toBeTruthy(); + expect(info.bash_script).toBeTruthy(); + }); + + test('update scorer', async () => { + const newType = uniqueName('sdk-scorer-updated'); + const updated = await scorer.update({ + type: newType, + bash_script: 'echo "score=0.5"', + }); + + expect(updated.id).toBe(scorerId); + expect(updated.type).toBe(newType); + }); + + // TODO: reenable this post-API fixes + // test('validate scorer', async () => { + // const result = await scorer.validate({ scoring_context: { test: true } }); + // expect(result.scoring_result).toBeDefined(); + // expect(typeof result.scoring_result.score).toBe('number'); + // }); + + test('get scorer by ID (ScorerOps.fromId)', async () => { + const retrieved = sdk.scorer.fromId(scorerId!); + expect(retrieved.id).toBe(scorerId); + + const info = await retrieved.getInfo(); + expect(info.id).toBe(scorerId); + }); + + test('list scorers (ScorerOps.list)', async () => { + const scorers = await sdk.scorer.list({ limit: 10 }); + expect(Array.isArray(scorers)).toBe(true); + expect(scorers.length).toBeGreaterThan(0); + expect(scorers.every((s) => typeof s.id === 'string' && s.id.length > 0)).toBe(true); + }); + }); +}); + + From febdac8822c709d7b9e59f19d093978921d2f3ee Mon Sep 17 00:00:00 2001 From: James Chainey Date: Thu, 18 Dec 2025 15:49:38 -0800 Subject: [PATCH 3/8] added missing static create method --- src/sdk.ts | 3 +-- src/sdk/scorer.ts | 21 +++++++++++++++++++ .../smoketests/object-oriented/scorer.test.ts | 15 +++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/sdk.ts b/src/sdk.ts index 2cf89563d..0c273c408 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -1286,8 +1286,7 @@ export class ScorerOps { * @returns {Promise} A {@link Scorer} instance */ async create(params: ScorerCreateParams, options?: Core.RequestOptions): Promise { - const response = await this.client.scenarios.scorers.create(params, options); - return Scorer.fromId(this.client, response.id); + return Scorer.create(this.client, params, options); } /** diff --git a/src/sdk/scorer.ts b/src/sdk/scorer.ts index f52b29e0a..4cafd75e9 100644 --- a/src/sdk/scorer.ts +++ b/src/sdk/scorer.ts @@ -1,6 +1,7 @@ import { Runloop } from '../index'; import type * as Core from '../core'; import type { + ScorerCreateParams, ScorerRetrieveResponse, ScorerUpdateResponse, ScorerUpdateParams, @@ -45,6 +46,26 @@ export class Scorer { this._id = scorerId; } + /** + * Create a new custom scorer. + * + * See the {@link ScorerOps.create} method for calling this + * @private + * + * @param {Runloop} client - The Runloop client instance + * @param {ScorerCreateParams} params - Scorer creation parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} A new {@link Scorer} instance + */ + static async create( + client: Runloop, + params: ScorerCreateParams, + options?: Core.RequestOptions, + ): Promise { + const scorerData = await client.scenarios.scorers.create(params, options); + return new Scorer(client, scorerData.id); + } + /** * Create a Scorer instance by ID without retrieving from API. * Use getInfo() to fetch the actual data when needed. diff --git a/tests/smoketests/object-oriented/scorer.test.ts b/tests/smoketests/object-oriented/scorer.test.ts index b5aeedfce..5abf60772 100644 --- a/tests/smoketests/object-oriented/scorer.test.ts +++ b/tests/smoketests/object-oriented/scorer.test.ts @@ -19,6 +19,21 @@ describe('smoketest: object-oriented scorers', () => { scorerId = scorer.id; }); + test('create scorer via Scorer.create (static)', async () => { + const created = await Scorer.create(sdk.api, { + type: uniqueName('sdk-scorer-static-create'), + bash_script: 'echo "score=1.0"', + }); + + expect(created).toBeDefined(); + expect(created.id).toBeTruthy(); + + const info = await created.getInfo(); + expect(info.id).toBe(created.id); + expect(info.type).toBeTruthy(); + expect(info.bash_script).toBeTruthy(); + }); + test('get scorer info', async () => { const info = await scorer.getInfo(); expect(info.id).toBe(scorerId); From b89a3d47e53d877fb2c01b26c97534a4d8aaa49b Mon Sep 17 00:00:00 2001 From: James Chainey Date: Thu, 18 Dec 2025 17:59:53 -0800 Subject: [PATCH 4/8] updated to reflect feedback --- README.md | 20 +++++ src/sdk.ts | 44 ++++++++++ tests/objects/scorer.test.ts | 57 ++++++++---- tests/sdk/scorer-ops.test.ts | 88 +++++++++++++++++++ .../smoketests/object-oriented/scorer.test.ts | 1 + 5 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 tests/sdk/scorer-ops.test.ts diff --git a/README.md b/README.md index 5f8f9364f..3bf407968 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ The SDK provides object-oriented interfaces for all major Runloop resources: - **[`runloop.blueprint`](https://runloopai.github.io/api-client-ts/stable/classes/BlueprintOps.html)** - Blueprint management (create, list, build blueprints) - **[`runloop.snapshot`](https://runloopai.github.io/api-client-ts/stable/classes/SnapshotOps.html)** - Snapshot management (list disk snapshots) - **[`runloop.storageObject`](https://runloopai.github.io/api-client-ts/stable/classes/StorageObjectOps.html)** - Storage object management (upload, download, list objects) +- **[`runloop.scorer`](https://runloopai.github.io/api-client-ts/stable/classes/ScorerOps.html)** - Scorer management (create, list, validate, update) - **[`runloop.api`](https://runloopai.github.io/api-client-ts/stable/classes/Runloop.html)** - Direct access to the REST API client ## TypeScript Support @@ -74,6 +75,25 @@ const runloop = new RunloopSDK(); const devbox: DevboxView = await runloop.devbox.create(); ``` +### Scorers + +Scorers are custom scoring functions used to evaluate scenario outputs. Create scorers via `runloop.scorer.create()`, then update or validate them with the returned `Scorer` instance: + +```typescript +import { RunloopSDK } from '@runloop/api-client'; + +const runloop = new RunloopSDK(); + +const scorer = await runloop.scorer.create({ + type: 'my_scorer', + bash_script: 'echo "score=1.0"', +}); + +await scorer.update({ bash_script: 'echo "score=0.5"' }); +const result = await scorer.validate({ scoring_context: { output: 'hello' } }); +console.log(result.scoring_result.score); +``` + ## Migration from API Client If you're currently using the legacy API, migration is straightforward: diff --git a/src/sdk.ts b/src/sdk.ts index 0c273c408..b1e1459d5 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -1271,6 +1271,50 @@ export class AgentOps { * Scorer SDK interface for managing custom scorers. * * @category Scorer + * + * @remarks + * ## Overview + * + * Scorers are custom scoring functions used to evaluate scenario outputs. A scorer is a + * script that runs and prints a score in the range [0.0, 1.0], e.g. `echo "score=0.5"`. + * + * ## Usage + * + * This interface is accessed via {@link RunloopSDK.scorer}. Create scorers with {@link ScorerOps.create} + * or reference an existing scorer by ID with {@link ScorerOps.fromId} to obtain a {@link Scorer} instance. + * + * @example + * ```typescript + * import { RunloopSDK } from '@runloop/api-client'; + * + * const runloop = new RunloopSDK(); + * + * // Create a scorer + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "score=1.0"', + * }); + * + * // Update the scorer + * await scorer.update({ bash_script: 'echo "score=0.5"' }); + * + * // Validate the scorer with a scoring context + * const result = await scorer.validate({ scoring_context: { output: 'hello' } }); + * console.log(result.scoring_result.score); + * ``` + * + * @example + * Get scorer info (typical usage): + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "1.0"', + * }); + * + * const info = await scorer.getInfo(); + * console.log(`Scorer ${info.id} (${info.type})`); + * ``` */ export class ScorerOps { /** diff --git a/tests/objects/scorer.test.ts b/tests/objects/scorer.test.ts index a6ec80bb7..4e73a803b 100644 --- a/tests/objects/scorer.test.ts +++ b/tests/objects/scorer.test.ts @@ -17,6 +17,7 @@ describe('Scorer', () => { mockClient = { scenarios: { scorers: { + create: jest.fn(), retrieve: jest.fn(), update: jest.fn(), validate: jest.fn(), @@ -25,7 +26,7 @@ describe('Scorer', () => { } as any; mockScorerData = { - id: 'scr-123', + id: 'sco-123', type: 'my_custom_scorer', bash_script: 'echo "score=1.0"', }; @@ -44,10 +45,28 @@ describe('Scorer', () => { describe('fromId', () => { it('should create a Scorer instance by ID without API call', () => { - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); expect(scorer).toBeInstanceOf(Scorer); - expect(scorer.id).toBe('scr-123'); + expect(scorer.id).toBe('sco-123'); + }); + }); + + describe('create', () => { + it('should create a scorer via API and return a Scorer instance', async () => { + mockClient.scenarios.scorers.create.mockResolvedValue(mockScorerData); + + const scorer = await Scorer.create(mockClient, { + type: 'my_custom_scorer', + bash_script: 'echo "score=1.0"', + }); + + expect(mockClient.scenarios.scorers.create).toHaveBeenCalledWith( + { type: 'my_custom_scorer', bash_script: 'echo "score=1.0"' }, + undefined, + ); + expect(scorer).toBeInstanceOf(Scorer); + expect(scorer.id).toBe('sco-123'); }); }); @@ -55,10 +74,10 @@ describe('Scorer', () => { it('should retrieve scorer information from API', async () => { mockClient.scenarios.scorers.retrieve.mockResolvedValue(mockScorerData); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); const info = await scorer.getInfo(); - expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('scr-123', undefined); + expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('sco-123', undefined); expect(info).toEqual(mockScorerData); expect(info.type).toBe('my_custom_scorer'); expect(info.bash_script).toBe('echo "score=1.0"'); @@ -67,10 +86,10 @@ describe('Scorer', () => { it('should pass options to the API client', async () => { mockClient.scenarios.scorers.retrieve.mockResolvedValue(mockScorerData); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); await scorer.getInfo({ timeout: 5000 }); - expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('scr-123', { timeout: 5000 }); + expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('sco-123', { timeout: 5000 }); }); }); @@ -83,14 +102,14 @@ describe('Scorer', () => { }; mockClient.scenarios.scorers.update.mockResolvedValue(updatedData); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); const result = await scorer.update({ type: 'my_custom_scorer_v2', bash_script: 'echo "score=0.5"', }); expect(mockClient.scenarios.scorers.update).toHaveBeenCalledWith( - 'scr-123', + 'sco-123', { type: 'my_custom_scorer_v2', bash_script: 'echo "score=0.5"' }, undefined, ); @@ -100,11 +119,11 @@ describe('Scorer', () => { it('should pass options to the API client', async () => { mockClient.scenarios.scorers.update.mockResolvedValue(mockScorerData); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); await scorer.update({ type: 'updated', bash_script: 'echo "score=1.0"' }, { timeout: 5000 }); expect(mockClient.scenarios.scorers.update).toHaveBeenCalledWith( - 'scr-123', + 'sco-123', { type: 'updated', bash_script: 'echo "score=1.0"' }, { timeout: 5000 }, ); @@ -115,13 +134,13 @@ describe('Scorer', () => { it('should validate scorer with scoring context', async () => { mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); const result = await scorer.validate({ scoring_context: { output: 'test output', expected: 'expected output' }, }); expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( - 'scr-123', + 'sco-123', { scoring_context: { output: 'test output', expected: 'expected output' } }, undefined, ); @@ -132,11 +151,11 @@ describe('Scorer', () => { it('should pass options to the API client', async () => { mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); await scorer.validate({ scoring_context: {} }, { timeout: 30000 }); expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( - 'scr-123', + 'sco-123', { scoring_context: {} }, { timeout: 30000 }, ); @@ -155,7 +174,7 @@ describe('Scorer', () => { }; mockClient.scenarios.scorers.validate.mockResolvedValue(lowScoreResult); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); const result = await scorer.validate({ scoring_context: { incorrect: true } }); expect(result.scoring_result.score).toBe(0.25); @@ -167,7 +186,7 @@ describe('Scorer', () => { const error = new Error('Scorer not found'); mockClient.scenarios.scorers.retrieve.mockRejectedValue(error); - const scorer = Scorer.fromId(mockClient, 'scr-nonexistent'); + const scorer = Scorer.fromId(mockClient, 'sco-nonexistent'); await expect(scorer.getInfo()).rejects.toThrow('Scorer not found'); }); @@ -176,7 +195,7 @@ describe('Scorer', () => { const error = new Error('Update failed'); mockClient.scenarios.scorers.update.mockRejectedValue(error); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); await expect(scorer.update({ type: 'new_type', bash_script: 'echo 1' })).rejects.toThrow('Update failed'); }); @@ -185,7 +204,7 @@ describe('Scorer', () => { const error = new Error('Validation timeout'); mockClient.scenarios.scorers.validate.mockRejectedValue(error); - const scorer = Scorer.fromId(mockClient, 'scr-123'); + const scorer = Scorer.fromId(mockClient, 'sco-123'); await expect(scorer.validate({ scoring_context: {} })).rejects.toThrow('Validation timeout'); }); diff --git a/tests/sdk/scorer-ops.test.ts b/tests/sdk/scorer-ops.test.ts new file mode 100644 index 000000000..195962e31 --- /dev/null +++ b/tests/sdk/scorer-ops.test.ts @@ -0,0 +1,88 @@ +import { ScorerOps } from '../../src/sdk'; +import { Scorer } from '../../src/sdk/scorer'; + +// Mock the Scorer class +jest.mock('../../src/sdk/scorer'); + +describe('ScorerOps', () => { + let mockClient: any; + let scorerOps: ScorerOps; + + beforeEach(() => { + jest.clearAllMocks(); + mockClient = { + scenarios: { + scorers: { + list: jest.fn(), + }, + }, + } as any; + + scorerOps = new ScorerOps(mockClient); + + // Note: We keep these ops-level tests even though they may feel duplicative with Scorer tests, + // because they cover what users call (`runloop.scorer.*`) and help ensure SDK-level coverage. + jest.spyOn(Scorer as any, 'fromId').mockImplementation((_client: any, id: string) => { + return { id } as unknown as Scorer; + }); + }); + + describe('create', () => { + it('should delegate to Scorer.create', async () => { + const mockScorerInstance = { id: 'sco-123', getInfo: jest.fn() } as unknown as Scorer; + jest.spyOn(Scorer as any, 'create').mockResolvedValue(mockScorerInstance); + + const scorer = await scorerOps.create( + { + type: 'my_scorer', + bash_script: 'echo "score=1.0"', + }, + undefined, + ); + + expect(Scorer.create).toHaveBeenCalledWith( + mockClient, + { + type: 'my_scorer', + bash_script: 'echo "score=1.0"', + }, + undefined, + ); + expect(scorer.id).toBe('sco-123'); + }); + }); + + describe('fromId', () => { + it('should delegate to Scorer.fromId', () => { + const scorer = scorerOps.fromId('sco-123'); + expect(Scorer.fromId).toHaveBeenCalledWith(mockClient, 'sco-123'); + expect(scorer.id).toBe('sco-123'); + }); + }); + + describe('list', () => { + it('should list scorers and return Scorer instances by ID', async () => { + const page = { + async *[Symbol.asyncIterator]() { + yield { id: 'sco-1' }; + yield { id: 'sco-2' }; + }, + } as any; + + mockClient.scenarios.scorers.list.mockResolvedValue(page); + + const scorers = await scorerOps.list({ limit: 2 }); + + expect(mockClient.scenarios.scorers.list).toHaveBeenCalledWith({ limit: 2 }, undefined); + expect(Scorer.fromId).toHaveBeenCalledWith(mockClient, 'sco-1'); + expect(Scorer.fromId).toHaveBeenCalledWith(mockClient, 'sco-2'); + + expect(Array.isArray(scorers)).toBe(true); + expect(scorers).toHaveLength(2); + expect(scorers[0].id).toBe('sco-1'); + expect(scorers[1].id).toBe('sco-2'); + }); + }); +}); + + diff --git a/tests/smoketests/object-oriented/scorer.test.ts b/tests/smoketests/object-oriented/scorer.test.ts index 5abf60772..8f2011aad 100644 --- a/tests/smoketests/object-oriented/scorer.test.ts +++ b/tests/smoketests/object-oriented/scorer.test.ts @@ -50,6 +50,7 @@ describe('smoketest: object-oriented scorers', () => { expect(updated.id).toBe(scorerId); expect(updated.type).toBe(newType); + expect(updated.bash_script).toBe('echo "score=0.5"'); }); // TODO: reenable this post-API fixes From abf9e2a62e23a39560046b18f11741554243155f Mon Sep 17 00:00:00 2001 From: James Chainey Date: Fri, 19 Dec 2025 14:12:50 -0800 Subject: [PATCH 5/8] improved example embedding --- src/sdk/scorer.ts | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/sdk/scorer.ts b/src/sdk/scorer.ts index 4cafd75e9..a1f0a8ac5 100644 --- a/src/sdk/scorer.ts +++ b/src/sdk/scorer.ts @@ -17,22 +17,20 @@ import type { * @remarks * ## Overview * - * The `Scorer` class provides a high-level API for managing custom scorers. + * The `Scorer` class provides a high-level, object-oriented API for managing custom scorers. * Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs. * - * ## Usage + * ## Quickstart * - * Obtain instances via `runloop.scorer.create()` or `runloop.scorer.fromId()`: - * - * @example * ```typescript + * import { RunloopSDK } from '@runloop/api-client-ts'; + * * const runloop = new RunloopSDK(); * const scorer = await runloop.scorer.create({ * type: 'my_scorer', - * bash_script: 'echo "score=1.0"' + * bash_script: 'echo "1.0"', * }); * - * // Validate the scorer * const result = await scorer.validate({ scoring_context: { output: 'test' } }); * console.log(`Score: ${result.scoring_result.score}`); * ``` @@ -49,9 +47,18 @@ export class Scorer { /** * Create a new custom scorer. * - * See the {@link ScorerOps.create} method for calling this + * See the {@link ScorerOps.create} method for calling this. * @private * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "1.0"', + * }); + * ``` + * * @param {Runloop} client - The Runloop client instance * @param {ScorerCreateParams} params - Scorer creation parameters * @param {Core.RequestOptions} [options] - Request options @@ -70,6 +77,17 @@ export class Scorer { * Create a Scorer instance by ID without retrieving from API. * Use getInfo() to fetch the actual data when needed. * + * See the {@link ScorerOps.fromId} method for calling this. + * @private + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = runloop.scorer.fromId('scs-123'); + * const info = await scorer.getInfo(); + * console.log(`Scorer type: ${info.type}`); + * ``` + * * @param {Runloop} client - The Runloop client instance * @param {string} id - The scorer ID * @returns {Scorer} A Scorer instance From fcc1ddf1b57b74d9f02ab6aa702314057f386679 Mon Sep 17 00:00:00 2001 From: James Chainey Date: Fri, 19 Dec 2025 14:40:40 -0800 Subject: [PATCH 6/8] feedback from review comments --- README.md | 4 +- src/sdk.ts | 42 ++++-- src/sdk/scorer.ts | 42 +++++- tests/objects/scorer.test.ts | 134 ++++++++++++------ tests/sdk/scorer-ops.test.ts | 38 ++--- .../smoketests/object-oriented/scorer.test.ts | 43 +++--- 6 files changed, 194 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 3bf407968..18789f626 100644 --- a/README.md +++ b/README.md @@ -86,10 +86,10 @@ const runloop = new RunloopSDK(); const scorer = await runloop.scorer.create({ type: 'my_scorer', - bash_script: 'echo "score=1.0"', + bash_script: 'echo "1.0"', }); -await scorer.update({ bash_script: 'echo "score=0.5"' }); +await scorer.update({ bash_script: 'echo "0.5"' }); const result = await scorer.validate({ scoring_context: { output: 'hello' } }); console.log(result.scoring_result.score); ``` diff --git a/src/sdk.ts b/src/sdk.ts index b1e1459d5..0494c2f6d 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -1276,7 +1276,7 @@ export class AgentOps { * ## Overview * * Scorers are custom scoring functions used to evaluate scenario outputs. A scorer is a - * script that runs and prints a score in the range [0.0, 1.0], e.g. `echo "score=0.5"`. + * script that runs and prints a score in the range [0.0, 1.0], e.g. `echo "0.5"`. * * ## Usage * @@ -1292,11 +1292,11 @@ export class AgentOps { * // Create a scorer * const scorer = await runloop.scorer.create({ * type: 'my_scorer', - * bash_script: 'echo "score=1.0"', + * bash_script: 'echo "1.0"', * }); * * // Update the scorer - * await scorer.update({ bash_script: 'echo "score=0.5"' }); + * await scorer.update({ bash_script: 'echo "0.5"' }); * * // Validate the scorer with a scoring context * const result = await scorer.validate({ scoring_context: { output: 'hello' } }); @@ -1325,6 +1325,18 @@ export class ScorerOps { /** * Create a new custom scorer. * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "1.0"', + * }); + * + * const info = await scorer.getInfo(); + * console.log(info.id); + * ``` + * * @param {ScorerCreateParams} params - Parameters for creating the scorer * @param {Core.RequestOptions} [options] - Request options * @returns {Promise} A {@link Scorer} instance @@ -1336,6 +1348,14 @@ export class ScorerOps { /** * Get a scorer object by its ID. * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = runloop.scorer.fromId('scs_123'); + * const info = await scorer.getInfo(); + * console.log(info.type); + * ``` + * * @param {string} id - The ID of the scorer * @returns {Scorer} A {@link Scorer} instance */ @@ -1346,19 +1366,19 @@ export class ScorerOps { /** * List all scorers with optional filters. * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorers = await runloop.scorer.list({ limit: 10 }); + * console.log(scorers.map((s) => s.id)); + * ``` + * * @param {ScorerListParams} [params] - Optional filter parameters * @param {Core.RequestOptions} [options] - Request options * @returns {Promise} An array of {@link Scorer} instances */ async list(params?: ScorerListParams, options?: Core.RequestOptions): Promise { - const page = await this.client.scenarios.scorers.list(params, options); - const scorers: Scorer[] = []; - - for await (const scorer of page) { - scorers.push(Scorer.fromId(this.client, scorer.id)); - } - - return scorers; + return Scorer.list(this.client, params, options); } } diff --git a/src/sdk/scorer.ts b/src/sdk/scorer.ts index a1f0a8ac5..00241fca0 100644 --- a/src/sdk/scorer.ts +++ b/src/sdk/scorer.ts @@ -2,6 +2,7 @@ import { Runloop } from '../index'; import type * as Core from '../core'; import type { ScorerCreateParams, + ScorerListParams, ScorerRetrieveResponse, ScorerUpdateResponse, ScorerUpdateParams, @@ -73,6 +74,43 @@ export class Scorer { return new Scorer(client, scorerData.id); } + /** + * List all scorers with optional filters. + * + * See the {@link ScorerOps.list} method for calling this. + * @private + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorers = await runloop.scorer.list({ limit: 10 }); + * + * for (const scorer of scorers) { + * const info = await scorer.getInfo(); + * console.log(`${info.id}: ${info.type}`); + * } + * ``` + * + * @param {Runloop} client - The Runloop client instance + * @param {ScorerListParams} [params] - Optional filter parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Array of {@link Scorer} instances + */ + static async list( + client: Runloop, + params?: ScorerListParams, + options?: Core.RequestOptions, + ): Promise { + const scorers = await client.scenarios.scorers.list(params, options); + const result: Scorer[] = []; + + for await (const scorer of scorers) { + result.push(Scorer.fromId(client, scorer.id)); + } + + return result; + } + /** * Create a Scorer instance by ID without retrieving from API. * Use getInfo() to fetch the actual data when needed. @@ -83,7 +121,7 @@ export class Scorer { * @example * ```typescript * const runloop = new RunloopSDK(); - * const scorer = runloop.scorer.fromId('scs-123'); + * const scorer = runloop.scorer.fromId('scs_123'); * const info = await scorer.getInfo(); * console.log(`Scorer type: ${info.type}`); * ``` @@ -127,7 +165,7 @@ export class Scorer { * ```typescript * const updated = await scorer.update({ * type: 'my_scorer_v2', - * bash_script: 'echo "score=0.5"' + * bash_script: 'echo "0.5"' * }); * ``` * diff --git a/tests/objects/scorer.test.ts b/tests/objects/scorer.test.ts index 4e73a803b..7636e305a 100644 --- a/tests/objects/scorer.test.ts +++ b/tests/objects/scorer.test.ts @@ -26,16 +26,16 @@ describe('Scorer', () => { } as any; mockScorerData = { - id: 'sco-123', + id: 'scs_123', type: 'my_custom_scorer', - bash_script: 'echo "score=1.0"', + bash_script: 'echo "1.0"', }; mockValidateResult = { name: 'my_custom_scorer', scoring_context: { output: 'test' }, scoring_result: { - output: 'score=1.0', + output: '1.0', score: 1.0, state: 'complete', scoring_function_name: 'test-scorer', @@ -45,10 +45,10 @@ describe('Scorer', () => { describe('fromId', () => { it('should create a Scorer instance by ID without API call', () => { - const scorer = Scorer.fromId(mockClient, 'sco-123'); + const scorer = Scorer.fromId(mockClient, 'scs_123'); expect(scorer).toBeInstanceOf(Scorer); - expect(scorer.id).toBe('sco-123'); + expect(scorer.id).toBe('scs_123'); }); }); @@ -58,15 +58,72 @@ describe('Scorer', () => { const scorer = await Scorer.create(mockClient, { type: 'my_custom_scorer', - bash_script: 'echo "score=1.0"', + bash_script: 'echo "1.0"', }); expect(mockClient.scenarios.scorers.create).toHaveBeenCalledWith( - { type: 'my_custom_scorer', bash_script: 'echo "score=1.0"' }, + { type: 'my_custom_scorer', bash_script: 'echo "1.0"' }, undefined, ); expect(scorer).toBeInstanceOf(Scorer); - expect(scorer.id).toBe('sco-123'); + expect(scorer.id).toBe('scs_123'); + }); + }); + + describe('list', () => { + it('should list scorers and return Scorer instances', async () => { + const mockScorers = [ + { id: 'scs_001', type: 'first', bash_script: 'echo "1.0"' }, + { id: 'scs_002', type: 'second', bash_script: 'echo "0.5"' }, + { id: 'scs_003', type: 'third', bash_script: 'echo "0.0"' }, + ]; + + const asyncIterator = { + async *[Symbol.asyncIterator]() { + for (const scorer of mockScorers) { + yield scorer; + } + }, + }; + + mockClient.scenarios.scorers.list.mockReturnValue(asyncIterator); + + const scorers = await Scorer.list(mockClient); + + expect(mockClient.scenarios.scorers.list).toHaveBeenCalledWith(undefined, undefined); + expect(scorers).toHaveLength(3); + expect(scorers[0]).toBeInstanceOf(Scorer); + expect(scorers[0]!.id).toBe('scs_001'); + expect(scorers[1]!.id).toBe('scs_002'); + expect(scorers[2]!.id).toBe('scs_003'); + }); + + it('should pass filter parameters to list', async () => { + const asyncIterator = { + async *[Symbol.asyncIterator]() { + yield { id: 'scs_001' }; + }, + }; + + mockClient.scenarios.scorers.list.mockReturnValue(asyncIterator); + + await Scorer.list(mockClient, { limit: 10 }); + + expect(mockClient.scenarios.scorers.list).toHaveBeenCalledWith({ limit: 10 }, undefined); + }); + + it('should handle empty list', async () => { + const asyncIterator = { + async *[Symbol.asyncIterator]() { + // Empty iterator + }, + }; + + mockClient.scenarios.scorers.list.mockReturnValue(asyncIterator); + + const scorers = await Scorer.list(mockClient); + + expect(scorers).toHaveLength(0); }); }); @@ -74,22 +131,22 @@ describe('Scorer', () => { it('should retrieve scorer information from API', async () => { mockClient.scenarios.scorers.retrieve.mockResolvedValue(mockScorerData); - const scorer = Scorer.fromId(mockClient, 'sco-123'); + const scorer = Scorer.fromId(mockClient, 'scs_123'); const info = await scorer.getInfo(); - expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('sco-123', undefined); + expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('scs_123', undefined); expect(info).toEqual(mockScorerData); expect(info.type).toBe('my_custom_scorer'); - expect(info.bash_script).toBe('echo "score=1.0"'); + expect(info.bash_script).toBe('echo "1.0"'); }); it('should pass options to the API client', async () => { mockClient.scenarios.scorers.retrieve.mockResolvedValue(mockScorerData); - const scorer = Scorer.fromId(mockClient, 'sco-123'); + const scorer = Scorer.fromId(mockClient, 'scs_123'); await scorer.getInfo({ timeout: 5000 }); - expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('sco-123', { timeout: 5000 }); + expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('scs_123', { timeout: 5000 }); }); }); @@ -98,19 +155,19 @@ describe('Scorer', () => { const updatedData: ScorerUpdateResponse = { ...mockScorerData, type: 'my_custom_scorer_v2', - bash_script: 'echo "score=0.5"', + bash_script: 'echo "0.5"', }; mockClient.scenarios.scorers.update.mockResolvedValue(updatedData); - const scorer = Scorer.fromId(mockClient, 'sco-123'); + const scorer = Scorer.fromId(mockClient, 'scs_123'); const result = await scorer.update({ type: 'my_custom_scorer_v2', - bash_script: 'echo "score=0.5"', + bash_script: 'echo "0.5"', }); expect(mockClient.scenarios.scorers.update).toHaveBeenCalledWith( - 'sco-123', - { type: 'my_custom_scorer_v2', bash_script: 'echo "score=0.5"' }, + 'scs_123', + { type: 'my_custom_scorer_v2', bash_script: 'echo "0.5"' }, undefined, ); expect(result.type).toBe('my_custom_scorer_v2'); @@ -119,12 +176,12 @@ describe('Scorer', () => { it('should pass options to the API client', async () => { mockClient.scenarios.scorers.update.mockResolvedValue(mockScorerData); - const scorer = Scorer.fromId(mockClient, 'sco-123'); - await scorer.update({ type: 'updated', bash_script: 'echo "score=1.0"' }, { timeout: 5000 }); + const scorer = Scorer.fromId(mockClient, 'scs_123'); + await scorer.update({ type: 'updated', bash_script: 'echo "1.0"' }, { timeout: 5000 }); expect(mockClient.scenarios.scorers.update).toHaveBeenCalledWith( - 'sco-123', - { type: 'updated', bash_script: 'echo "score=1.0"' }, + 'scs_123', + { type: 'updated', bash_script: 'echo "1.0"' }, { timeout: 5000 }, ); }); @@ -134,51 +191,34 @@ describe('Scorer', () => { it('should validate scorer with scoring context', async () => { mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); - const scorer = Scorer.fromId(mockClient, 'sco-123'); + const scorer = Scorer.fromId(mockClient, 'scs_123'); const result = await scorer.validate({ scoring_context: { output: 'test output', expected: 'expected output' }, }); expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( - 'sco-123', + 'scs_123', { scoring_context: { output: 'test output', expected: 'expected output' } }, undefined, ); expect(result.scoring_result.score).toBe(1.0); - expect(result.scoring_result.output).toBe('score=1.0'); + expect(result.scoring_result.output).toBe('1.0'); }); it('should pass options to the API client', async () => { mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); - const scorer = Scorer.fromId(mockClient, 'sco-123'); + const scorer = Scorer.fromId(mockClient, 'scs_123'); await scorer.validate({ scoring_context: {} }, { timeout: 30000 }); expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( - 'sco-123', + 'scs_123', { scoring_context: {} }, { timeout: 30000 }, ); }); - it('should handle low score validation result', async () => { - const lowScoreResult: ScorerValidateResponse = { - name: 'my_custom_scorer', - scoring_context: { incorrect: true }, - scoring_result: { - output: 'score=0.25', - score: 0.25, - state: 'complete', - scoring_function_name: 'test-scorer', - }, - }; - mockClient.scenarios.scorers.validate.mockResolvedValue(lowScoreResult); - - const scorer = Scorer.fromId(mockClient, 'sco-123'); - const result = await scorer.validate({ scoring_context: { incorrect: true } }); - expect(result.scoring_result.score).toBe(0.25); - }); }); describe('error handling', () => { @@ -186,7 +226,7 @@ describe('Scorer', () => { const error = new Error('Scorer not found'); mockClient.scenarios.scorers.retrieve.mockRejectedValue(error); - const scorer = Scorer.fromId(mockClient, 'sco-nonexistent'); + const scorer = Scorer.fromId(mockClient, 'scs_nonexistent'); await expect(scorer.getInfo()).rejects.toThrow('Scorer not found'); }); @@ -195,7 +235,7 @@ describe('Scorer', () => { const error = new Error('Update failed'); mockClient.scenarios.scorers.update.mockRejectedValue(error); - const scorer = Scorer.fromId(mockClient, 'sco-123'); + const scorer = Scorer.fromId(mockClient, 'scs_123'); await expect(scorer.update({ type: 'new_type', bash_script: 'echo 1' })).rejects.toThrow('Update failed'); }); @@ -204,7 +244,7 @@ describe('Scorer', () => { const error = new Error('Validation timeout'); mockClient.scenarios.scorers.validate.mockRejectedValue(error); - const scorer = Scorer.fromId(mockClient, 'sco-123'); + const scorer = Scorer.fromId(mockClient, 'scs_123'); await expect(scorer.validate({ scoring_context: {} })).rejects.toThrow('Validation timeout'); }); diff --git a/tests/sdk/scorer-ops.test.ts b/tests/sdk/scorer-ops.test.ts index 195962e31..ee47db3da 100644 --- a/tests/sdk/scorer-ops.test.ts +++ b/tests/sdk/scorer-ops.test.ts @@ -22,20 +22,21 @@ describe('ScorerOps', () => { // Note: We keep these ops-level tests even though they may feel duplicative with Scorer tests, // because they cover what users call (`runloop.scorer.*`) and help ensure SDK-level coverage. - jest.spyOn(Scorer as any, 'fromId').mockImplementation((_client: any, id: string) => { + jest.spyOn(Scorer as any, 'fromId').mockImplementation((...args: any[]) => { + const id = args[1] as string; return { id } as unknown as Scorer; }); }); describe('create', () => { it('should delegate to Scorer.create', async () => { - const mockScorerInstance = { id: 'sco-123', getInfo: jest.fn() } as unknown as Scorer; + const mockScorerInstance = { id: 'scs_123', getInfo: jest.fn() } as unknown as Scorer; jest.spyOn(Scorer as any, 'create').mockResolvedValue(mockScorerInstance); const scorer = await scorerOps.create( { type: 'my_scorer', - bash_script: 'echo "score=1.0"', + bash_script: 'echo "1.0"', }, undefined, ); @@ -44,43 +45,32 @@ describe('ScorerOps', () => { mockClient, { type: 'my_scorer', - bash_script: 'echo "score=1.0"', + bash_script: 'echo "1.0"', }, undefined, ); - expect(scorer.id).toBe('sco-123'); + expect(scorer.id).toBe('scs_123'); }); }); describe('fromId', () => { it('should delegate to Scorer.fromId', () => { - const scorer = scorerOps.fromId('sco-123'); - expect(Scorer.fromId).toHaveBeenCalledWith(mockClient, 'sco-123'); - expect(scorer.id).toBe('sco-123'); + const scorer = scorerOps.fromId('scs_123'); + expect(Scorer.fromId).toHaveBeenCalledWith(mockClient, 'scs_123'); + expect(scorer.id).toBe('scs_123'); }); }); describe('list', () => { - it('should list scorers and return Scorer instances by ID', async () => { - const page = { - async *[Symbol.asyncIterator]() { - yield { id: 'sco-1' }; - yield { id: 'sco-2' }; - }, - } as any; - - mockClient.scenarios.scorers.list.mockResolvedValue(page); + it('should delegate to Scorer.list', async () => { + jest.spyOn(Scorer as any, 'list').mockResolvedValue([{ id: 'scs_1' }, { id: 'scs_2' }] as unknown as Scorer[]); const scorers = await scorerOps.list({ limit: 2 }); - expect(mockClient.scenarios.scorers.list).toHaveBeenCalledWith({ limit: 2 }, undefined); - expect(Scorer.fromId).toHaveBeenCalledWith(mockClient, 'sco-1'); - expect(Scorer.fromId).toHaveBeenCalledWith(mockClient, 'sco-2'); - - expect(Array.isArray(scorers)).toBe(true); + expect(Scorer.list).toHaveBeenCalledWith(mockClient, { limit: 2 }, undefined); expect(scorers).toHaveLength(2); - expect(scorers[0].id).toBe('sco-1'); - expect(scorers[1].id).toBe('sco-2'); + expect(scorers[0]!.id).toBe('scs_1'); + expect(scorers[1]!.id).toBe('scs_2'); }); }); }); diff --git a/tests/smoketests/object-oriented/scorer.test.ts b/tests/smoketests/object-oriented/scorer.test.ts index 8f2011aad..7d3a77890 100644 --- a/tests/smoketests/object-oriented/scorer.test.ts +++ b/tests/smoketests/object-oriented/scorer.test.ts @@ -7,58 +7,54 @@ describe('smoketest: object-oriented scorers', () => { describe('scorer lifecycle', () => { let scorer: Scorer; let scorerId: string | undefined; + let scorerType: string; beforeAll(async () => { // Create a scorer first (no delete endpoint currently, so we keep it small + uniquely named) + scorerType = uniqueName('sdk-scorer'); scorer = await sdk.scorer.create({ - type: uniqueName('sdk-scorer'), - bash_script: 'echo "score=1.0"', + type: scorerType, + bash_script: 'echo "1.0"', }); expect(scorer).toBeDefined(); - expect(scorer.id).toBeTruthy(); + expect(scorer.id).toMatch(/^scs_/); scorerId = scorer.id; }); test('create scorer via Scorer.create (static)', async () => { + const type = uniqueName('sdk-scorer-static-create'); + const bashScript = 'echo "1.0"'; const created = await Scorer.create(sdk.api, { - type: uniqueName('sdk-scorer-static-create'), - bash_script: 'echo "score=1.0"', + type, + bash_script: bashScript, }); expect(created).toBeDefined(); - expect(created.id).toBeTruthy(); + expect(created.id).toMatch(/^scs_/); const info = await created.getInfo(); expect(info.id).toBe(created.id); - expect(info.type).toBeTruthy(); - expect(info.bash_script).toBeTruthy(); - }); - - test('get scorer info', async () => { - const info = await scorer.getInfo(); - expect(info.id).toBe(scorerId); - expect(info.type).toBeTruthy(); - expect(info.bash_script).toBeTruthy(); + expect(info.type).toBe(type); + expect(info.bash_script).toBe(bashScript); }); test('update scorer', async () => { const newType = uniqueName('sdk-scorer-updated'); const updated = await scorer.update({ type: newType, - bash_script: 'echo "score=0.5"', + bash_script: 'echo "0.5"', }); expect(updated.id).toBe(scorerId); expect(updated.type).toBe(newType); - expect(updated.bash_script).toBe('echo "score=0.5"'); + expect(updated.bash_script).toBe('echo "0.5"'); }); - // TODO: reenable this post-API fixes - // test('validate scorer', async () => { - // const result = await scorer.validate({ scoring_context: { test: true } }); - // expect(result.scoring_result).toBeDefined(); - // expect(typeof result.scoring_result.score).toBe('number'); - // }); + test('validate scorer', async () => { + const result = await scorer.validate({ scoring_context: { test: true } }); + expect(result.scoring_result).toBeDefined(); + expect(typeof result.scoring_result.score).toBe('number'); + }); test('get scorer by ID (ScorerOps.fromId)', async () => { const retrieved = sdk.scorer.fromId(scorerId!); @@ -66,6 +62,7 @@ describe('smoketest: object-oriented scorers', () => { const info = await retrieved.getInfo(); expect(info.id).toBe(scorerId); + expect(info.type).toBe(scorerType); }); test('list scorers (ScorerOps.list)', async () => { From 0bf83884b39915563f636d59ad914f902e03c9d7 Mon Sep 17 00:00:00 2001 From: James Chainey Date: Fri, 19 Dec 2025 14:41:57 -0800 Subject: [PATCH 7/8] add mock for create --- tests/objects/scorer.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/objects/scorer.test.ts b/tests/objects/scorer.test.ts index 7636e305a..d48af9117 100644 --- a/tests/objects/scorer.test.ts +++ b/tests/objects/scorer.test.ts @@ -21,6 +21,7 @@ describe('Scorer', () => { retrieve: jest.fn(), update: jest.fn(), validate: jest.fn(), + list: jest.fn(), }, }, } as any; From a83225e8a8f0a6ac6fbbe0dc7dc6a1a8cd7ca136 Mon Sep 17 00:00:00 2001 From: James Chainey Date: Fri, 19 Dec 2025 15:20:55 -0800 Subject: [PATCH 8/8] extra tests for coverage --- tests/smoketests/object-oriented/scorer.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/smoketests/object-oriented/scorer.test.ts b/tests/smoketests/object-oriented/scorer.test.ts index 7d3a77890..4592d4166 100644 --- a/tests/smoketests/object-oriented/scorer.test.ts +++ b/tests/smoketests/object-oriented/scorer.test.ts @@ -56,10 +56,13 @@ describe('smoketest: object-oriented scorers', () => { expect(typeof result.scoring_result.score).toBe('number'); }); - test('get scorer by ID (ScorerOps.fromId)', async () => { + test('get scorer by ID', async () => { const retrieved = sdk.scorer.fromId(scorerId!); expect(retrieved.id).toBe(scorerId); + const staticRetrieved = Scorer.fromId(sdk.api, scorerId!); + expect(staticRetrieved.id).toBe(scorerId); + const info = await retrieved.getInfo(); expect(info.id).toBe(scorerId); expect(info.type).toBe(scorerType); @@ -71,6 +74,7 @@ describe('smoketest: object-oriented scorers', () => { expect(scorers.length).toBeGreaterThan(0); expect(scorers.every((s) => typeof s.id === 'string' && s.id.length > 0)).toBe(true); }); + }); });