SDK update merge from dev

Author: reisepass

peerBenchJS/packages/sdk/.DS_Store

@@ -18,50 +18,72 @@ Scorers are responsible for taking a Response that is made for a Prompt and givi
 
 ### Abstract Methods
 
-**`scoreOne(response: PromptResponse, options?: Record<string, any>): Promise<number | undefined>`**
+**`scoreOne(response: PromptResponse, options?: Record<string, any>): MaybePromise<PromptScore | undefined>`**
 
 - This is the main scoring method you must implement
 - Takes a `PromptResponse` object containing both the Prompt and Response
 - Optional `options` parameter for configurable scoring behavior
-- Must return a number between 0 and 1, or `undefined` if scoring fails
-- The number represents the score: 1.0 = perfect, 0.0 = completely wrong
+- Must return a `PromptScore` object or `undefined` if scoring fails
+- The `PromptScore` object includes the original response data plus a `score` field (number between 0 and 1)
+- The score represents: 1.0 = perfect, 0.0 = completely wrong
+- The `PromptScore` structure extends `PromptResponse` with additional fields:
+  - `score?: number` - The calculated score (0-1)
+  - `metadata?: Record<string, any>` - Additional metadata about the scoring
 
-**`canScore(response: PromptResponse): Promise<boolean>`**
+**`canScore(response: PromptResponse): MaybePromise<boolean>`**
 
 - This method determines if your Scorer can handle a specific Response
 - Returns `true` if your Scorer implementation is eligible to score the Response and `false` if it isn't.
+- `MaybePromise<T>` means the method can return either `T` or `Promise<T>` (synchronous or asynchronous)
 
 Here's a very simple scorer implementation that demonstrates the basic structure:
 
 ```typescript
 import { AbstractScorer } from "@/scorers/abstract/abstract-scorer";
-import { PromptResponse } from "@/types";
+import { PromptResponse, PromptScore } from "@/types";
 
 export class SimpleExactMatchScorer extends AbstractScorer {
   readonly identifier = "simple-exact-match";
 
   async scoreOne(
     response: PromptResponse,
     options?: Record<string, any>
-  ): Promise<number | undefined> {
+  ): Promise<PromptScore | undefined> {
+    if (!(await this.canScore(response))) {
+      return undefined;
+    }
+
     // Extract the correct answer from the prompt
     const correctAnswer = response.prompt.answer;
 
     // Get the model's response
-    const modelResponse = response.response.content;
+    const modelResponse = response.data;
 
     // Simple exact match scoring (case-insensitive)
     const isCorrect =
-      correctAnswer.toLowerCase().trim() === modelResponse.toLowerCase().trim();
-
-    // Return 1.0 for correct answers, 0.0 for incorrect
-    return isCorrect ? 1.0 : 0.0;
+      correctAnswer?.toLowerCase().trim() ===
+      modelResponse?.toLowerCase().trim();
+
+    // Calculate score: 1.0 for correct answers, 0.0 for incorrect
+    const score = isCorrect ? 1.0 : 0.0;
+
+    // Return PromptScore object with the original response data plus score and metadata
+    return {
+      ...response,
+      score,
+      metadata: {
+        scorerIdentifier: this.identifier,
+        isExactMatch: isCorrect,
+      },
+    } as PromptScore;
   }
 
   async canScore(response: PromptResponse): Promise<boolean> {
-    // This scorer can handle any response that has content
+    // This scorer can handle any response that has data and a prompt with an answer
     return (
-      response.response.content && typeof response.response.content === "string"
+      response.data !== undefined &&
+      response.prompt !== undefined &&
+      response.prompt.answer !== undefined
     );
   }
 }
@@ -105,7 +127,7 @@ async function main() {
       metadata: {},
       scorers: ["simple-exact-match"],
     },
-    data: "A", // Correct answer
+    data: "Paris", // Correct answer
     provider: "mock",
     modelId: "mock-model",
     modelName: "Mock Model",
@@ -124,7 +146,7 @@ async function main() {
 
   const wrongResponse = {
     ...mockResponse,
-    data: "C", // Wrong answer
+    data: "London", // Wrong answer
   };
 
   // Test if the Scorer can handle these responses
@@ -135,13 +157,17 @@ async function main() {
   console.log("Can score wrong response:", canScoreWrong);
 
   if (canScoreCorrect) {
-    const correctScore = await scorer.scoreOne(mockResponse);
-    console.log("Correct answer score:", correctScore);
+    const correctScoreResult = await scorer.scoreOne(mockResponse);
+    console.log("Correct answer score result:", correctScoreResult);
+    console.log("Score:", correctScoreResult?.score);
+    console.log("Metadata:", correctScoreResult?.metadata);
   }
 
   if (canScoreWrong) {
-    const wrongScore = await scorer.scoreOne(wrongResponse);
-    console.log("Wrong answer score:", wrongScore);
+    const wrongScoreResult = await scorer.scoreOne(wrongResponse);
+    console.log("Wrong answer score result:", wrongScoreResult);
+    console.log("Score:", wrongScoreResult?.score);
+    console.log("Metadata:", wrongScoreResult?.metadata);
   }
 }
 
@@ -160,9 +186,10 @@ This will execute `dev.ts`.
 
 ## Examples
 
-See the `examples/scorers/` directory for complete working examples:
+See the `src/scorers/` directory for complete working examples:
 
-- `exact-match-scorer.ts` - Exact match scoring
-- `semantic-similarity-scorer.ts` - Semantic similarity scoring
+- `exact-match-scorer.ts` - Exact match scoring that returns PromptScore objects
+- `multiple-choice-scorer.ts` - Multiple choice scoring with answer extraction
+- `ref-answer-equality-llm-judge-scorer.ts` - LLM-based scoring for reference answer equality
 
-These examples demonstrate real-world implementations and can serve as templates for your own Scorers.
+These examples demonstrate real-world implementations and show how to properly return `PromptScore` objects with scores and metadata. They can serve as templates for your own Scorers.

peerBenchJS/packages/sdk/docs/implementing-a-scorer.md

@@ -3,11 +3,11 @@ import tseslint from "typescript-eslint";
 
 export default [
   {
-    ignores: ["dist/**/*.js"],
+    ignores: ["dist/**/*.js", "examples/**/*.ts"],
   },
   ...tseslint.config({
     extends: [eslint.configs.recommended, tseslint.configs.recommended],
-    files: ["src/**/*.ts", "examples/**/*.ts", "scripts/**/*.ts"],
+    files: ["src/**/*.ts", "scripts/**/*.ts"],
 
     rules: {
       "@typescript-eslint/no-explicit-any": "off",

peerBenchJS/packages/sdk/eslint.config.js

@@ -27,24 +27,8 @@ export class NewsAPICollector extends AbstractCollector<NewsArticle[]> {
   readonly identifier = "news-api";
 
   // API configuration - these would be set during initialization
-  private apiKey: string; // Authentication key for the API
   private baseUrl = "https://newsapi.org/v2"; // Base URL for API endpoints
 
-  /**
-   * Constructor for the NewsAPI collector
-   *
-   * @param apiKey - Authentication key required to access the news API
-   *
-   * The constructor demonstrates how to:
-   * - Accept configuration parameters
-   * - Store sensitive information securely
-   * - Initialize the collector with required credentials
-   */
-  constructor(apiKey: string) {
-    super(); // Call parent constructor
-    this.apiKey = apiKey; // Store the API key for later use
-  }
-
   /**
    * Main collection method - fetches news articles based on search query
    *
@@ -69,10 +53,12 @@ export class NewsAPICollector extends AbstractCollector<NewsArticle[]> {
     searchQuery: string,
     options: {
       includeSensitiveContent?: boolean; // Option to include sensitive content
-    } = {
-      includeSensitiveContent: false, // Default to excluding sensitive content
+      apiKey: string;
     }
   ): Promise<NewsArticle[] | undefined> {
+    // Default to excluding sensitive content
+    const includeSensitiveContent = options.includeSensitiveContent ?? false;
+
     // Type guard: ensure the source is a string search query
     // This prevents runtime errors from invalid input types
     if (typeof searchQuery !== "string") {
@@ -83,8 +69,8 @@ export class NewsAPICollector extends AbstractCollector<NewsArticle[]> {
     // URLSearchParams automatically handles URL encoding and parameter formatting
     const params = new URLSearchParams();
     params.set("q", searchQuery); // Set the search query
-    params.set("apiKey", this.apiKey); // Add authentication
-    if (options.includeSensitiveContent) {
+    params.set("apiKey", options.apiKey); // Add authentication
+    if (includeSensitiveContent) {
       params.set("sensitive", "true"); // Optional parameter for content filtering
     }
 

peerBenchJS/packages/sdk/examples/collectors/news-api-collector.ts

@@ -25,6 +25,10 @@
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
+    },
+    "./generators/pubmed/helpers": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
     }
   },
   "files": [
@@ -40,17 +44,22 @@
   "dependencies": {
     "@inquirer/prompts": "^7.4.1",
     "@supabase/supabase-js": "^2.56.0",
+    "@types/sbd": "^1.0.5",
     "abort-controller": "^3.0.0",
     "ansis": "^3.17.0",
     "axios": "^1.11.0",
     "cheerio": "^1.1.2",
     "commander": "^13.1.0",
     "csv": "^6.3.11",
+    "decimal.js": "^10.6.0",
     "fast-xml-parser": "^5.2.5",
     "glob": "^11.0.2",
     "hyparquet": "^1.11.0",
+    "json-stable-stringify": "^1.3.0",
+    "jsonrepair": "^3.13.0",
     "multiformats": "^13.3.2",
     "openai": "^4.92.1",
+    "sbd": "^1.0.19",
     "table": "^6.9.0",
     "uuid": "^11.1.0",
     "viem": "^2.26.2",