peerbenchJS releae 0.0.16

Author: reisepass

peerBenchJS/Dockerfile.webapp

@@ -0,0 +1,50 @@
+FROM node:20-bookworm AS base
+
+FROM base AS deps
+WORKDIR /app
+
+# Copy package.json files of all repos
+COPY package*.json .
+COPY apps/webapp/package*.json ./apps/webapp/
+COPY packages/sdk/package*.json ./packages/sdk/
+
+# Install dependencies
+RUN npm ci
+
+# Rebuild the source code only when needed
+FROM base AS builder
+WORKDIR /app
+COPY --from=deps /app/node_modules ./node_modules
+COPY --from=deps /app/apps/webapp/node_modules ./apps/webapp/node_modules
+COPY --from=deps /app/packages/sdk/node_modules ./packages/sdk/node_modules
+COPY . .
+
+# Disable telemetry
+ENV NEXT_TELEMETRY_DISABLED=1
+
+# Build the repo including webapp and sdk
+RUN npm run build
+
+# Production image, copy all the files and run next
+FROM base AS runner
+WORKDIR /app
+
+ENV NODE_ENV=production
+ENV NEXT_TELEMETRY_DISABLED=1
+
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nextjs
+
+COPY --from=builder /app/apps/webapp/public ./public
+
+COPY --from=builder --chown=nextjs:nodejs /app/apps/webapp/.next/standalone ./
+COPY --from=builder --chown=nextjs:nodejs /app/apps/webapp/.next/static ./.next/static
+
+USER nextjs
+
+EXPOSE 3000
+
+ENV PORT=3000
+ENV HOSTNAME="0.0.0.0"
+
+CMD ["node", "apps/webapp/server.js"]

peerBenchJS/README.md

@@ -1,9 +1,8 @@
 
 
-
 # The problem:   
 
-Public benchmark test data sets make AI model performance comparable. But this creates an incentivization for closed source models to cheat the benchmarks by training on test data or creating heuristics that overfit the benchmark test dataset .....
+Public benchmark test data sets make AI model performance comparable. But this creates an incentivization for closed source models to cheat the benchmarks by training on test data or creating heuristics that overfit the benchmark test dataset .
 
 # Our solution:
 

peerBenchJS/apps/cli/src/collectors.ts

@@ -0,0 +1,18 @@
+import {
+  AbstractCollector,
+  PubMedCollector,
+  StringCollector,
+} from "@peerbench/sdk";
+
+export const collectors: AbstractCollector<unknown>[] = [
+  new StringCollector(),
+  new PubMedCollector(),
+  // TODO: Add more Collectors if there are anymore
+];
+
+/**
+ * Gets a Collector by its identifier from the given list of Collectors
+ */
+export function getCollector(identifier: string) {
+  return collectors.find((c) => c.identifier === identifier);
+}

peerBenchJS/apps/cli/src/commands/collect/index.ts

@@ -0,0 +1,119 @@
+import { program } from "@/core/program";
+import { getCollector } from "@/collectors";
+import { AbstractCollector } from "@peerbench/sdk";
+import { saveCollectedData } from "./save-collected-data";
+import { logger } from "@/core/logger";
+import { hashFile } from "@/utils/hash-file";
+import { signFile } from "@/utils/sign-file";
+import { FileSchema } from "@/validation/file-schema";
+import { JSONSchema } from "@/validation/json-schema";
+import { z } from "zod";
+import fs from "fs/promises";
+
+export const collectCommand = program
+  .command("collect")
+  .description("Collects raw data")
+  .requiredOption(
+    "-s, --source <sources...>",
+    "Source inputs for the Collector"
+  )
+  .requiredOption(
+    "-o, --output <output>",
+    "Output directory",
+    "peerbench-data/collected"
+  )
+  .requiredOption(
+    "-c, --collector <collector>",
+    "Collector identifier",
+    (value) => {
+      const collector = getCollector(value);
+      if (!collector) {
+        throw new Error(`Collector "${value}" not found`);
+      }
+      return collector;
+    }
+  )
+  .option(
+    "--options <path or JSON string>",
+    "Path to the options file or a JSON string that will be passed to the Collector",
+    (value) =>
+      z
+        .union([
+          JSONSchema<any>(),
+          FileSchema().pipe(
+            JSONSchema({
+              message: "Given options file is not a valid JSON file",
+            })
+          ),
+        ])
+        .parse(value)
+  )
+  .option(
+    "--args <path or JSON string>",
+    "Path to the initialization args file or a JSON string that will be passed to the Collector",
+    (value) =>
+      z
+        .union([
+          JSONSchema<any>(),
+          FileSchema().pipe(
+            JSONSchema({ message: "Given args file is not a valid JSON file" })
+          ),
+        ])
+        .parse(value)
+  )
+  .option("-t, --tag <tags...>", "Tags to be attached to the output file")
+  .action(
+    async (options: {
+      source: string[];
+      collector: AbstractCollector<unknown>;
+      output: string;
+      tag: string[];
+      options: any;
+      args: any;
+    }) => {
+      // Ensure the output directory exists
+      await fs.mkdir(options.output, { recursive: true });
+
+      const tags = options.tag;
+
+      // Initialize the Collector
+      await options.collector.initialize(options.args);
+
+      for (const source of options.source) {
+        logger.info(
+          `Collecting data from ${source} using ${options.collector.identifier}`
+        );
+
+        const collectedData = await options.collector
+          .collect(source, options.options)
+          .catch((err) => {
+            logger.error(
+              `Error collecting data from ${source} using ${options.collector.identifier}: ${err}`
+            );
+
+            return undefined;
+          });
+
+        // Collected data will be undefined if
+        // the Collector failed to collect the data
+        if (!collectedData) {
+          throw new Error("Failed to collect data");
+        }
+
+        // Save the data
+        const filePath = await saveCollectedData({
+          outputDirectory: options.output,
+          collectorIdentifier: options.collector.identifier,
+          source,
+          collectedData,
+          tags,
+        });
+
+        // Hash and sign the output file
+        await hashFile(filePath);
+        await signFile(`${filePath}.cid`); // Only sign the hash file
+
+        logger.info(`Data saved to: ${filePath}`);
+      }
+    }
+  );

peerBenchJS/apps/cli/src/commands/collect/save-collected-data.ts

@@ -0,0 +1,35 @@
+import { dateString } from "@/utils/date-string";
+import { normalizePath } from "@/utils/normalize-path";
+import { CollectedDataSave } from "@/validation/collected-data-schema";
+import fs from "fs/promises";
+import path from "path";
+
+/**
+ * Saves the data that was collected by a Collector to a file in JSON format
+ *
+ * @returns The path to the saved file
+ */
+export async function saveCollectedData(params: {
+  source: unknown;
+  outputDirectory: string;
+  collectorIdentifier: string;
+  collectedData: any;
+  tags?: string[];
+}) {
+  const tags =
+    params.tags && params.tags.length > 0 ? `.${params.tags.join("-")}` : "";
+  const timestamp = dateString();
+  const fileName = normalizePath(
+    `${params.collectorIdentifier}.${timestamp}.collected${tags}.json`
+  );
+  const filePath = path.join(params.outputDirectory, fileName);
+  const fileContent: CollectedDataSave = {
+    collectorIdentifier: params.collectorIdentifier,
+    source: params.source,
+    data: params.collectedData,
+  };
+
+  await fs.writeFile(filePath, JSON.stringify(fileContent, null, 2), "utf-8");
+
+  return filePath;
+}

peerBenchJS/apps/cli/src/commands/generate/index.ts

@@ -0,0 +1,125 @@
+import { logger } from "@/core/logger";
+import { program } from "@/core/program";
+import { getGenerator } from "@/generators";
+import {
+  CollectedDataSave,
+  CollectedDataSchema,
+} from "@/validation/collected-data-schema";
+import { FileSchema } from "@/validation/file-schema";
+import { JSONSchema } from "@/validation/json-schema";
+import { AbstractGenerator } from "@peerbench/sdk";
+import { z } from "zod";
+import { savePrompts } from "./save-prompts";
+import { hashFile } from "@/utils/hash-file";
+import { signFile } from "@/utils/sign-file";
+import { OptionOutputDir } from "../options/output-dir";
+import { OptionTags } from "../options/tags";
+import fs from "fs/promises";
+
+program
+  .command("generate")
+  .description("Generates new Prompts")
+  .requiredOption(
+    "-g, --generator <identifier>",
+    "Identifier of the Generator",
+    (value) => {
+      const generator = getGenerator(value);
+      if (!generator) {
+        throw new Error(`Generator "${value}" not found`);
+      }
+      return generator;
+    }
+  )
+  .requiredOption(
+    "-f, --file <path>",
+    "Path to the collected data file",
+    (value) =>
+      FileSchema({ message: "Data file doesn't exist" })
+        .pipe(
+          JSONSchema({
+            message: "Given data file is invalid",
+            schema: CollectedDataSchema,
+          })
+        )
+        .parse(value)
+  )
+  .addOption(OptionOutputDir("generated").makeOptionMandatory(true))
+  .option(
+    "--options <path or JSON string>",
+    "Path to the options file or a JSON string that will be passed to the Generator",
+    (value) =>
+      z
+        .union([
+          JSONSchema<any>(),
+          FileSchema().pipe(
+            JSONSchema({
+              message: "Given options file is invalid",
+              schema: z.record(z.string(), z.unknown()),
+            })
+          ),
+        ])
+        .parse(value)
+  )
+  .option(
+    "--args <path or JSON string>",
+    "Path to the initialization args file or a JSON string that will be passed to the Generator",
+    (value) =>
+      z
+        .union([
+          JSONSchema<any>(),
+          FileSchema().pipe(
+            JSONSchema({
+              message: "Given args file is invalid",
+              schema: z.array(z.unknown()),
+            })
+          ),
+        ])
+        .parse(value)
+  )
+  .addOption(OptionTags())
+  .action(
+    async (options: {
+      generator: AbstractGenerator;
+      file: CollectedDataSave;
+      tags: string[];
+      options: Record<string, unknown>;
+      args: unknown[];
+      output: string;
+    }) => {
+      // Ensure the output directory exists
+      await fs.mkdir(options.output, { recursive: true });
+
+      logger.info("Initializing Generator");
+
+      // Initialize the Generator
+      await options.generator.initialize(options.args);
+
+      logger.info(`Checking the Generator eligibility for the given file`);
+      const canHandle = await options.generator.canHandle(options.file);
+      if (!canHandle) {
+        throw new Error("Generator doesn't support the given file");
+      }
+
+      logger.info(`Generating Prompts`);
+      const prompts = await options.generator.generate(
+        options.file.data,
+        options.options
+      );
+      logger.info(`Prompts generated successfully`);
+
+      logger.info(`Saving Prompts`);
+      // Save the data
+      const filePath = await savePrompts({
+        generatorIdentifier: options.generator.identifier,
+        outputDirectory: options.output,
+        prompts,
+        tags: options.tags,
+      });
+
+      // Hash and sign the output file
+      await hashFile(filePath);
+      await signFile(`${filePath}.cid`); // Only sign the hash file
+
+      logger.info(`Prompts saved to: ${filePath}`);
+    }
+  );

peerBenchJS/apps/cli/src/commands/generate/save-prompts.ts

@@ -0,0 +1,29 @@
+import { Prompt } from "@peerbench/sdk";
+import path from "path";
+import fs from "fs/promises";
+import { dateString } from "@/utils/date-string";
+import { normalizePath } from "@/utils/normalize-path";
+
+/**
+ * Saves the Prompts to a file in peerBench Task format
+ *
+ * @returns The path to the saved file
+ */
+export async function savePrompts(params: {
+  generatorIdentifier: string;
+  outputDirectory: string;
+  prompts: Prompt[];
+  tags?: string[];
+}) {
+  const tags =
+    params.tags && params.tags.length > 0 ? `.${params.tags.join("-")}` : "";
+  const timestamp = dateString();
+  const fileName = normalizePath(
+    `${params.generatorIdentifier}.${timestamp}.generated${tags}.json`
+  );
+  const filePath = path.join(params.outputDirectory, fileName);
+
+  await fs.writeFile(filePath, JSON.stringify(params.prompts));
+
+  return filePath;
+}