diff --git a/README.md b/README.md index 5f8f9364f..18789f626 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ The SDK provides object-oriented interfaces for all major Runloop resources: - **[`runloop.blueprint`](https://runloopai.github.io/api-client-ts/stable/classes/BlueprintOps.html)** - Blueprint management (create, list, build blueprints) - **[`runloop.snapshot`](https://runloopai.github.io/api-client-ts/stable/classes/SnapshotOps.html)** - Snapshot management (list disk snapshots) - **[`runloop.storageObject`](https://runloopai.github.io/api-client-ts/stable/classes/StorageObjectOps.html)** - Storage object management (upload, download, list objects) +- **[`runloop.scorer`](https://runloopai.github.io/api-client-ts/stable/classes/ScorerOps.html)** - Scorer management (create, list, validate, update) - **[`runloop.api`](https://runloopai.github.io/api-client-ts/stable/classes/Runloop.html)** - Direct access to the REST API client ## TypeScript Support @@ -74,6 +75,25 @@ const runloop = new RunloopSDK(); const devbox: DevboxView = await runloop.devbox.create(); ``` +### Scorers + +Scorers are custom scoring functions used to evaluate scenario outputs. Create scorers via `runloop.scorer.create()`, then update or validate them with the returned `Scorer` instance: + +```typescript +import { RunloopSDK } from '@runloop/api-client'; + +const runloop = new RunloopSDK(); + +const scorer = await runloop.scorer.create({ + type: 'my_scorer', + bash_script: 'echo "1.0"', +}); + +await scorer.update({ bash_script: 'echo "0.5"' }); +const result = await scorer.validate({ scoring_context: { output: 'hello' } }); +console.log(result.scoring_result.score); +``` + ## Migration from API Client If you're currently using the legacy API, migration is straightforward: diff --git a/src/sdk.ts b/src/sdk.ts index 8bf069e59..0494c2f6d 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -6,6 +6,7 @@ import { Blueprint, type CreateParams as BlueprintCreateParams } from './sdk/blu import { Snapshot } from './sdk/snapshot'; import { StorageObject } from './sdk/storage-object'; import { Agent } from './sdk/agent'; +import { Scorer } from './sdk/scorer'; // Import types used in this file import type { @@ -17,6 +18,7 @@ import type { import type { BlueprintListParams } from './resources/blueprints'; import type { ObjectCreateParams, ObjectListParams } from './resources/objects'; import type { AgentCreateParams, AgentListParams } from './resources/agents'; +import type { ScorerCreateParams, ScorerListParams } from './resources/scenarios/scorers'; import { PollingOptions } from './lib/polling'; import * as Shared from './resources/shared'; @@ -190,6 +192,7 @@ type ContentType = ObjectCreateParams['content_type']; * - `snapshot` - {@link SnapshotOps} * - `storageObject` - {@link StorageObjectOps} * - `agent` - {@link AgentOps} + * - `scorer` - {@link ScorerOps} * * See the documentation for each Operations class for more details. * @@ -260,6 +263,14 @@ export class RunloopSDK { */ public readonly agent: AgentOps; + /** + * **Scorer Operations** - {@link ScorerOps} for creating and accessing {@link Scorer} class instances. + * + * Scorers are custom scoring functions that evaluate scenario outputs. They define scripts + * that produce a score in the range [0.0, 1.0] for scenario runs. + */ + public readonly scorer: ScorerOps; + /** * Creates a new RunloopSDK instance. * @param {ClientOptions} [options] - Optional client configuration options. @@ -271,6 +282,7 @@ export class RunloopSDK { this.snapshot = new SnapshotOps(this.api); this.storageObject = new StorageObjectOps(this.api); this.agent = new AgentOps(this.api); + this.scorer = new ScorerOps(this.api); } } @@ -1255,6 +1267,121 @@ export class AgentOps { } } +/** + * Scorer SDK interface for managing custom scorers. + * + * @category Scorer + * + * @remarks + * ## Overview + * + * Scorers are custom scoring functions used to evaluate scenario outputs. A scorer is a + * script that runs and prints a score in the range [0.0, 1.0], e.g. `echo "0.5"`. + * + * ## Usage + * + * This interface is accessed via {@link RunloopSDK.scorer}. Create scorers with {@link ScorerOps.create} + * or reference an existing scorer by ID with {@link ScorerOps.fromId} to obtain a {@link Scorer} instance. + * + * @example + * ```typescript + * import { RunloopSDK } from '@runloop/api-client'; + * + * const runloop = new RunloopSDK(); + * + * // Create a scorer + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "1.0"', + * }); + * + * // Update the scorer + * await scorer.update({ bash_script: 'echo "0.5"' }); + * + * // Validate the scorer with a scoring context + * const result = await scorer.validate({ scoring_context: { output: 'hello' } }); + * console.log(result.scoring_result.score); + * ``` + * + * @example + * Get scorer info (typical usage): + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "1.0"', + * }); + * + * const info = await scorer.getInfo(); + * console.log(`Scorer ${info.id} (${info.type})`); + * ``` + */ +export class ScorerOps { + /** + * @private + */ + constructor(private client: RunloopAPI) {} + + /** + * Create a new custom scorer. + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "1.0"', + * }); + * + * const info = await scorer.getInfo(); + * console.log(info.id); + * ``` + * + * @param {ScorerCreateParams} params - Parameters for creating the scorer + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} A {@link Scorer} instance + */ + async create(params: ScorerCreateParams, options?: Core.RequestOptions): Promise { + return Scorer.create(this.client, params, options); + } + + /** + * Get a scorer object by its ID. + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = runloop.scorer.fromId('scs_123'); + * const info = await scorer.getInfo(); + * console.log(info.type); + * ``` + * + * @param {string} id - The ID of the scorer + * @returns {Scorer} A {@link Scorer} instance + */ + fromId(id: string): Scorer { + return Scorer.fromId(this.client, id); + } + + /** + * List all scorers with optional filters. + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorers = await runloop.scorer.list({ limit: 10 }); + * console.log(scorers.map((s) => s.id)); + * ``` + * + * @param {ScorerListParams} [params] - Optional filter parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} An array of {@link Scorer} instances + */ + async list(params?: ScorerListParams, options?: Core.RequestOptions): Promise { + return Scorer.list(this.client, params, options); + } +} + // @deprecated Use {@link RunloopSDK} instead. /** * @deprecated Use {@link RunloopSDK} instead. @@ -1275,11 +1402,13 @@ export declare namespace RunloopSDK { SnapshotOps as SnapshotOps, StorageObjectOps as StorageObjectOps, AgentOps as AgentOps, + ScorerOps as ScorerOps, Devbox as Devbox, Blueprint as Blueprint, Snapshot as Snapshot, StorageObject as StorageObject, Agent as Agent, + Scorer as Scorer, }; } // Export SDK classes from sdk/sdk.ts - these are separate from RunloopSDK to avoid circular dependencies @@ -1293,6 +1422,7 @@ export { Snapshot, StorageObject, Agent, + Scorer, Execution, ExecutionResult, } from './sdk/index'; diff --git a/src/sdk/index.ts b/src/sdk/index.ts index 65ce3bf1d..38cb0ce25 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -5,3 +5,4 @@ export { StorageObject } from './storage-object'; export { Agent } from './agent'; export { Execution } from './execution'; export { ExecutionResult } from './execution-result'; +export { Scorer } from './scorer'; diff --git a/src/sdk/scorer.ts b/src/sdk/scorer.ts new file mode 100644 index 000000000..00241fca0 --- /dev/null +++ b/src/sdk/scorer.ts @@ -0,0 +1,202 @@ +import { Runloop } from '../index'; +import type * as Core from '../core'; +import type { + ScorerCreateParams, + ScorerListParams, + ScorerRetrieveResponse, + ScorerUpdateResponse, + ScorerUpdateParams, + ScorerValidateParams, + ScorerValidateResponse, +} from '../resources/scenarios/scorers'; + +/** + * Object-oriented interface for working with custom Scorers. + * + * @category Scorer + * + * @remarks + * ## Overview + * + * The `Scorer` class provides a high-level, object-oriented API for managing custom scorers. + * Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs. + * + * ## Quickstart + * + * ```typescript + * import { RunloopSDK } from '@runloop/api-client-ts'; + * + * const runloop = new RunloopSDK(); + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "1.0"', + * }); + * + * const result = await scorer.validate({ scoring_context: { output: 'test' } }); + * console.log(`Score: ${result.scoring_result.score}`); + * ``` + */ +export class Scorer { + private client: Runloop; + private _id: string; + + private constructor(client: Runloop, scorerId: string) { + this.client = client; + this._id = scorerId; + } + + /** + * Create a new custom scorer. + * + * See the {@link ScorerOps.create} method for calling this. + * @private + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = await runloop.scorer.create({ + * type: 'my_scorer', + * bash_script: 'echo "1.0"', + * }); + * ``` + * + * @param {Runloop} client - The Runloop client instance + * @param {ScorerCreateParams} params - Scorer creation parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} A new {@link Scorer} instance + */ + static async create( + client: Runloop, + params: ScorerCreateParams, + options?: Core.RequestOptions, + ): Promise { + const scorerData = await client.scenarios.scorers.create(params, options); + return new Scorer(client, scorerData.id); + } + + /** + * List all scorers with optional filters. + * + * See the {@link ScorerOps.list} method for calling this. + * @private + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorers = await runloop.scorer.list({ limit: 10 }); + * + * for (const scorer of scorers) { + * const info = await scorer.getInfo(); + * console.log(`${info.id}: ${info.type}`); + * } + * ``` + * + * @param {Runloop} client - The Runloop client instance + * @param {ScorerListParams} [params] - Optional filter parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Array of {@link Scorer} instances + */ + static async list( + client: Runloop, + params?: ScorerListParams, + options?: Core.RequestOptions, + ): Promise { + const scorers = await client.scenarios.scorers.list(params, options); + const result: Scorer[] = []; + + for await (const scorer of scorers) { + result.push(Scorer.fromId(client, scorer.id)); + } + + return result; + } + + /** + * Create a Scorer instance by ID without retrieving from API. + * Use getInfo() to fetch the actual data when needed. + * + * See the {@link ScorerOps.fromId} method for calling this. + * @private + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scorer = runloop.scorer.fromId('scs_123'); + * const info = await scorer.getInfo(); + * console.log(`Scorer type: ${info.type}`); + * ``` + * + * @param {Runloop} client - The Runloop client instance + * @param {string} id - The scorer ID + * @returns {Scorer} A Scorer instance + */ + static fromId(client: Runloop, id: string): Scorer { + return new Scorer(client, id); + } + + /** + * Get the scorer ID. + * @returns {string} The scorer ID + */ + get id(): string { + return this._id; + } + + /** + * Fetch current scorer details from the API. + * + * @example + * ```typescript + * const info = await scorer.getInfo(); + * console.log(`Scorer type: ${info.type}`); + * ``` + * + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Current scorer details + */ + async getInfo(options?: Core.RequestOptions): Promise { + return this.client.scenarios.scorers.retrieve(this._id, options); + } + + /** + * Update the scorer's type or bash script. + * + * @example + * ```typescript + * const updated = await scorer.update({ + * type: 'my_scorer_v2', + * bash_script: 'echo "0.5"' + * }); + * ``` + * + * @param {ScorerUpdateParams} params - Update parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Updated scorer details + */ + async update(params: ScorerUpdateParams, options?: Core.RequestOptions): Promise { + return this.client.scenarios.scorers.update(this._id, params, options); + } + + /** + * Run the scorer against the provided context and return the result. + * + * @example + * ```typescript + * const result = await scorer.validate({ + * scoring_context: { output: 'test output', expected: 'test output' } + * }); + * console.log(`Validation score: ${result.scoring_result.score}`); + * console.log(`Output: ${result.scoring_result.output}`); + * ``` + * + * @param {ScorerValidateParams} params - Validation parameters + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Validation result with score + */ + async validate( + params: ScorerValidateParams, + options?: Core.RequestOptions, + ): Promise { + return this.client.scenarios.scorers.validate(this._id, params, options); + } +} diff --git a/src/types.ts b/src/types.ts index c4f12af36..a4a1a60ed 100644 --- a/src/types.ts +++ b/src/types.ts @@ -86,6 +86,7 @@ export type * from './resources/secrets'; // ============================================================================= export type * from './resources/scenarios/scenarios'; +export type * from './resources/scenarios/scorers'; // ============================================================================= // Benchmark Types diff --git a/tests/objects/scorer.test.ts b/tests/objects/scorer.test.ts new file mode 100644 index 000000000..d48af9117 --- /dev/null +++ b/tests/objects/scorer.test.ts @@ -0,0 +1,254 @@ +import { Scorer } from '../../src/sdk/scorer'; +import type { + ScorerRetrieveResponse, + ScorerUpdateResponse, + ScorerValidateResponse, +} from '../../src/resources/scenarios/scorers'; + +// Mock the Runloop client +jest.mock('../../src/index'); + +describe('Scorer', () => { + let mockClient: any; + let mockScorerData: ScorerRetrieveResponse; + let mockValidateResult: ScorerValidateResponse; + + beforeEach(() => { + mockClient = { + scenarios: { + scorers: { + create: jest.fn(), + retrieve: jest.fn(), + update: jest.fn(), + validate: jest.fn(), + list: jest.fn(), + }, + }, + } as any; + + mockScorerData = { + id: 'scs_123', + type: 'my_custom_scorer', + bash_script: 'echo "1.0"', + }; + + mockValidateResult = { + name: 'my_custom_scorer', + scoring_context: { output: 'test' }, + scoring_result: { + output: '1.0', + score: 1.0, + state: 'complete', + scoring_function_name: 'test-scorer', + }, + }; + }); + + describe('fromId', () => { + it('should create a Scorer instance by ID without API call', () => { + const scorer = Scorer.fromId(mockClient, 'scs_123'); + + expect(scorer).toBeInstanceOf(Scorer); + expect(scorer.id).toBe('scs_123'); + }); + }); + + describe('create', () => { + it('should create a scorer via API and return a Scorer instance', async () => { + mockClient.scenarios.scorers.create.mockResolvedValue(mockScorerData); + + const scorer = await Scorer.create(mockClient, { + type: 'my_custom_scorer', + bash_script: 'echo "1.0"', + }); + + expect(mockClient.scenarios.scorers.create).toHaveBeenCalledWith( + { type: 'my_custom_scorer', bash_script: 'echo "1.0"' }, + undefined, + ); + expect(scorer).toBeInstanceOf(Scorer); + expect(scorer.id).toBe('scs_123'); + }); + }); + + describe('list', () => { + it('should list scorers and return Scorer instances', async () => { + const mockScorers = [ + { id: 'scs_001', type: 'first', bash_script: 'echo "1.0"' }, + { id: 'scs_002', type: 'second', bash_script: 'echo "0.5"' }, + { id: 'scs_003', type: 'third', bash_script: 'echo "0.0"' }, + ]; + + const asyncIterator = { + async *[Symbol.asyncIterator]() { + for (const scorer of mockScorers) { + yield scorer; + } + }, + }; + + mockClient.scenarios.scorers.list.mockReturnValue(asyncIterator); + + const scorers = await Scorer.list(mockClient); + + expect(mockClient.scenarios.scorers.list).toHaveBeenCalledWith(undefined, undefined); + expect(scorers).toHaveLength(3); + expect(scorers[0]).toBeInstanceOf(Scorer); + expect(scorers[0]!.id).toBe('scs_001'); + expect(scorers[1]!.id).toBe('scs_002'); + expect(scorers[2]!.id).toBe('scs_003'); + }); + + it('should pass filter parameters to list', async () => { + const asyncIterator = { + async *[Symbol.asyncIterator]() { + yield { id: 'scs_001' }; + }, + }; + + mockClient.scenarios.scorers.list.mockReturnValue(asyncIterator); + + await Scorer.list(mockClient, { limit: 10 }); + + expect(mockClient.scenarios.scorers.list).toHaveBeenCalledWith({ limit: 10 }, undefined); + }); + + it('should handle empty list', async () => { + const asyncIterator = { + async *[Symbol.asyncIterator]() { + // Empty iterator + }, + }; + + mockClient.scenarios.scorers.list.mockReturnValue(asyncIterator); + + const scorers = await Scorer.list(mockClient); + + expect(scorers).toHaveLength(0); + }); + }); + + describe('getInfo', () => { + it('should retrieve scorer information from API', async () => { + mockClient.scenarios.scorers.retrieve.mockResolvedValue(mockScorerData); + + const scorer = Scorer.fromId(mockClient, 'scs_123'); + const info = await scorer.getInfo(); + + expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('scs_123', undefined); + expect(info).toEqual(mockScorerData); + expect(info.type).toBe('my_custom_scorer'); + expect(info.bash_script).toBe('echo "1.0"'); + }); + + it('should pass options to the API client', async () => { + mockClient.scenarios.scorers.retrieve.mockResolvedValue(mockScorerData); + + const scorer = Scorer.fromId(mockClient, 'scs_123'); + await scorer.getInfo({ timeout: 5000 }); + + expect(mockClient.scenarios.scorers.retrieve).toHaveBeenCalledWith('scs_123', { timeout: 5000 }); + }); + }); + + describe('update', () => { + it('should update scorer with provided parameters', async () => { + const updatedData: ScorerUpdateResponse = { + ...mockScorerData, + type: 'my_custom_scorer_v2', + bash_script: 'echo "0.5"', + }; + mockClient.scenarios.scorers.update.mockResolvedValue(updatedData); + + const scorer = Scorer.fromId(mockClient, 'scs_123'); + const result = await scorer.update({ + type: 'my_custom_scorer_v2', + bash_script: 'echo "0.5"', + }); + + expect(mockClient.scenarios.scorers.update).toHaveBeenCalledWith( + 'scs_123', + { type: 'my_custom_scorer_v2', bash_script: 'echo "0.5"' }, + undefined, + ); + expect(result.type).toBe('my_custom_scorer_v2'); + }); + + it('should pass options to the API client', async () => { + mockClient.scenarios.scorers.update.mockResolvedValue(mockScorerData); + + const scorer = Scorer.fromId(mockClient, 'scs_123'); + await scorer.update({ type: 'updated', bash_script: 'echo "1.0"' }, { timeout: 5000 }); + + expect(mockClient.scenarios.scorers.update).toHaveBeenCalledWith( + 'scs_123', + { type: 'updated', bash_script: 'echo "1.0"' }, + { timeout: 5000 }, + ); + }); + }); + + describe('validate', () => { + it('should validate scorer with scoring context', async () => { + mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); + + const scorer = Scorer.fromId(mockClient, 'scs_123'); + const result = await scorer.validate({ + scoring_context: { output: 'test output', expected: 'expected output' }, + }); + + expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( + 'scs_123', + { scoring_context: { output: 'test output', expected: 'expected output' } }, + undefined, + ); + expect(result.scoring_result.score).toBe(1.0); + expect(result.scoring_result.output).toBe('1.0'); + }); + + it('should pass options to the API client', async () => { + mockClient.scenarios.scorers.validate.mockResolvedValue(mockValidateResult); + + const scorer = Scorer.fromId(mockClient, 'scs_123'); + await scorer.validate({ scoring_context: {} }, { timeout: 30000 }); + + expect(mockClient.scenarios.scorers.validate).toHaveBeenCalledWith( + 'scs_123', + { scoring_context: {} }, + { timeout: 30000 }, + ); + }); + + + }); + + describe('error handling', () => { + it('should handle API errors on getInfo', async () => { + const error = new Error('Scorer not found'); + mockClient.scenarios.scorers.retrieve.mockRejectedValue(error); + + const scorer = Scorer.fromId(mockClient, 'scs_nonexistent'); + + await expect(scorer.getInfo()).rejects.toThrow('Scorer not found'); + }); + + it('should handle API errors on update', async () => { + const error = new Error('Update failed'); + mockClient.scenarios.scorers.update.mockRejectedValue(error); + + const scorer = Scorer.fromId(mockClient, 'scs_123'); + + await expect(scorer.update({ type: 'new_type', bash_script: 'echo 1' })).rejects.toThrow('Update failed'); + }); + + it('should handle API errors on validate', async () => { + const error = new Error('Validation timeout'); + mockClient.scenarios.scorers.validate.mockRejectedValue(error); + + const scorer = Scorer.fromId(mockClient, 'scs_123'); + + await expect(scorer.validate({ scoring_context: {} })).rejects.toThrow('Validation timeout'); + }); + }); +}); + diff --git a/tests/sdk/scorer-ops.test.ts b/tests/sdk/scorer-ops.test.ts new file mode 100644 index 000000000..ee47db3da --- /dev/null +++ b/tests/sdk/scorer-ops.test.ts @@ -0,0 +1,78 @@ +import { ScorerOps } from '../../src/sdk'; +import { Scorer } from '../../src/sdk/scorer'; + +// Mock the Scorer class +jest.mock('../../src/sdk/scorer'); + +describe('ScorerOps', () => { + let mockClient: any; + let scorerOps: ScorerOps; + + beforeEach(() => { + jest.clearAllMocks(); + mockClient = { + scenarios: { + scorers: { + list: jest.fn(), + }, + }, + } as any; + + scorerOps = new ScorerOps(mockClient); + + // Note: We keep these ops-level tests even though they may feel duplicative with Scorer tests, + // because they cover what users call (`runloop.scorer.*`) and help ensure SDK-level coverage. + jest.spyOn(Scorer as any, 'fromId').mockImplementation((...args: any[]) => { + const id = args[1] as string; + return { id } as unknown as Scorer; + }); + }); + + describe('create', () => { + it('should delegate to Scorer.create', async () => { + const mockScorerInstance = { id: 'scs_123', getInfo: jest.fn() } as unknown as Scorer; + jest.spyOn(Scorer as any, 'create').mockResolvedValue(mockScorerInstance); + + const scorer = await scorerOps.create( + { + type: 'my_scorer', + bash_script: 'echo "1.0"', + }, + undefined, + ); + + expect(Scorer.create).toHaveBeenCalledWith( + mockClient, + { + type: 'my_scorer', + bash_script: 'echo "1.0"', + }, + undefined, + ); + expect(scorer.id).toBe('scs_123'); + }); + }); + + describe('fromId', () => { + it('should delegate to Scorer.fromId', () => { + const scorer = scorerOps.fromId('scs_123'); + expect(Scorer.fromId).toHaveBeenCalledWith(mockClient, 'scs_123'); + expect(scorer.id).toBe('scs_123'); + }); + }); + + describe('list', () => { + it('should delegate to Scorer.list', async () => { + jest.spyOn(Scorer as any, 'list').mockResolvedValue([{ id: 'scs_1' }, { id: 'scs_2' }] as unknown as Scorer[]); + + const scorers = await scorerOps.list({ limit: 2 }); + + expect(Scorer.list).toHaveBeenCalledWith(mockClient, { limit: 2 }, undefined); + expect(scorers).toHaveLength(2); + expect(scorers[0]!.id).toBe('scs_1'); + expect(scorers[1]!.id).toBe('scs_2'); + }); + }); +}); + + diff --git a/tests/smoketests/object-oriented/scorer.test.ts b/tests/smoketests/object-oriented/scorer.test.ts new file mode 100644 index 000000000..4592d4166 --- /dev/null +++ b/tests/smoketests/object-oriented/scorer.test.ts @@ -0,0 +1,81 @@ +import { makeClientSDK, uniqueName } from '../utils'; +import { Scorer } from '@runloop/api-client/sdk'; + +const sdk = makeClientSDK(); + +describe('smoketest: object-oriented scorers', () => { + describe('scorer lifecycle', () => { + let scorer: Scorer; + let scorerId: string | undefined; + let scorerType: string; + + beforeAll(async () => { + // Create a scorer first (no delete endpoint currently, so we keep it small + uniquely named) + scorerType = uniqueName('sdk-scorer'); + scorer = await sdk.scorer.create({ + type: scorerType, + bash_script: 'echo "1.0"', + }); + expect(scorer).toBeDefined(); + expect(scorer.id).toMatch(/^scs_/); + scorerId = scorer.id; + }); + + test('create scorer via Scorer.create (static)', async () => { + const type = uniqueName('sdk-scorer-static-create'); + const bashScript = 'echo "1.0"'; + const created = await Scorer.create(sdk.api, { + type, + bash_script: bashScript, + }); + + expect(created).toBeDefined(); + expect(created.id).toMatch(/^scs_/); + + const info = await created.getInfo(); + expect(info.id).toBe(created.id); + expect(info.type).toBe(type); + expect(info.bash_script).toBe(bashScript); + }); + + test('update scorer', async () => { + const newType = uniqueName('sdk-scorer-updated'); + const updated = await scorer.update({ + type: newType, + bash_script: 'echo "0.5"', + }); + + expect(updated.id).toBe(scorerId); + expect(updated.type).toBe(newType); + expect(updated.bash_script).toBe('echo "0.5"'); + }); + + test('validate scorer', async () => { + const result = await scorer.validate({ scoring_context: { test: true } }); + expect(result.scoring_result).toBeDefined(); + expect(typeof result.scoring_result.score).toBe('number'); + }); + + test('get scorer by ID', async () => { + const retrieved = sdk.scorer.fromId(scorerId!); + expect(retrieved.id).toBe(scorerId); + + const staticRetrieved = Scorer.fromId(sdk.api, scorerId!); + expect(staticRetrieved.id).toBe(scorerId); + + const info = await retrieved.getInfo(); + expect(info.id).toBe(scorerId); + expect(info.type).toBe(scorerType); + }); + + test('list scorers (ScorerOps.list)', async () => { + const scorers = await sdk.scorer.list({ limit: 10 }); + expect(Array.isArray(scorers)).toBe(true); + expect(scorers.length).toBeGreaterThan(0); + expect(scorers.every((s) => typeof s.id === 'string' && s.id.length > 0)).toBe(true); + }); + + }); +}); + + diff --git a/tests/smoketests/object-oriented/sdk.test.ts b/tests/smoketests/object-oriented/sdk.test.ts index c7c5ce2c6..f086ff48c 100644 --- a/tests/smoketests/object-oriented/sdk.test.ts +++ b/tests/smoketests/object-oriented/sdk.test.ts @@ -10,6 +10,7 @@ describe('smoketest: object-oriented SDK', () => { expect(sdk.blueprint).toBeDefined(); expect(sdk.snapshot).toBeDefined(); expect(sdk.storageObject).toBeDefined(); + expect(sdk.scorer).toBeDefined(); expect(sdk.api).toBeDefined(); });