Allow autoevals to support both zod 3 and zod 4 (#155)

## Changes
Allow autoevals to install either zod 3 or zod 4.

The typescript sdk was updated to allow zod to be a peer dependency in
order to work with either zod 3 or zod 4.

This PR makes a similar update to the autoevals package and runs a
matrix with zod 3 and zod 4 over the existing tests.

Our internal uses of autoevals does not allow for a direct upgrade to
zod v4 at this time but the peer dependency should unblock users use of
using autoevals with zod 4.

---------

Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: cpinn

.github/workflows/js.yaml

@@ -7,11 +7,15 @@ on:
 
 jobs:
   build:
+    name: node ${{ matrix.node-version }} / zod ${{ matrix.zod }}
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
         node-version: [20.x, 22.x, 24.x]
+        # zod is a peerDependency ("^3.25.0 || ^4.0.0"); test both majors.
+        zod: ["3", "4"]
 
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -25,6 +29,14 @@ jobs:
           node-version: ${{ matrix.node-version }}
           cache: pnpm
       - run: pnpm install --frozen-lockfile
+      # The lockfile pins zod 3, so override across the whole tree
+      - name: Force zod 4
+        if: matrix.zod == '4'
+        run: |
+          printf '\noverrides:\n  zod: ^4\n' >> pnpm-workspace.yaml
+          pnpm install --no-frozen-lockfile
+      - name: Verify resolved zod major
+        run: node -e "const v=require('zod/package.json').version; console.log('zod', v); if (!v.startsWith('${{ matrix.zod }}.')) { console.error('expected zod ${{ matrix.zod }}.x'); process.exit(1); }"
       - run: pnpm run test
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

evals/package.json

@@ -13,12 +13,15 @@
   "license": "ISC",
   "dependencies": {
     "autoevals": "workspace:*",
-    "braintrust": "^0.0.140",
-    "zod": "^3.22.4"
+    "braintrust": "2.0.1"
   },
   "devDependencies": {
     "@types/node": "^20.10.5",
     "duckdb": "^1.0.0",
-    "tsx": "^3.14.0"
+    "tsx": "^3.14.0",
+    "zod": "3.25.67"
+  },
+  "peerDependencies": {
+    "zod": "^3.25.0 || ^4.0.0"
   }
 }

evals/src/autoevals.eval.ts

@@ -7,7 +7,7 @@ import {
   coqaCaseSchema,
   dataDir,
 } from "./datasets";
-import { z } from "zod";
+import { z } from "zod/v3";
 import {
   AnswerCorrectness,
   ClosedQA,

evals/src/datasets.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v3";
 
 import path from "path";
 

evals/src/sync_datasets.ts

@@ -1,6 +1,6 @@
 import { duckq, getDuckDBConn } from "./duckdb";
 
-import { z } from "zod";
+import { z } from "zod/v3";
 import {
   coqaSchema,
   dataDir,

js/ragas.ts

@@ -106,10 +106,10 @@ import { LLMArgs } from "./llm";
 import { getDefaultModel, getDefaultEmbeddingModel } from "./oai";
 import { buildOpenAIClient, extractOpenAIArgs } from "./oai";
 import OpenAI from "openai";
+import { zodFunction } from "openai/helpers/zod";
 import { ListContains } from "./list";
 import { EmbeddingSimilarity } from "./string";
-import { z } from "zod";
-import zodToJsonSchema from "zod-to-json-schema";
+import { z } from "zod/v3";
 import { makePartial, ScorerWithPartial } from "./partial";
 
 type RagasArgs = {
@@ -188,14 +188,11 @@ export const ContextEntityRecall: ScorerWithPartial<
       },
     ],
     tools: [
-      {
-        type: "function",
-        function: {
-          name: "extract_entities",
-          description: "Extract unique entities from a given text",
-          parameters: zodToJsonSchema(entitySchema),
-        },
-      },
+      zodFunction({
+        name: "extract_entities",
+        description: "Extract unique entities from a given text",
+        parameters: entitySchema,
+      }),
     ],
     tool_choice: { type: "function", function: { name: "extract_entities" } },
   });
@@ -268,14 +265,11 @@ export const ContextRelevancy: ScorerWithPartial<string, RagasArgs> =
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "extract_sentences",
-            description: "Extract relevant sentences from a given context",
-            parameters: zodToJsonSchema(relevantSentencesSchema),
-          },
-        },
+        zodFunction({
+          name: "extract_sentences",
+          description: "Extract relevant sentences from a given context",
+          parameters: relevantSentencesSchema,
+        }),
       ],
       tool_choice: {
         type: "function",
@@ -371,13 +365,10 @@ export const ContextRecall: ScorerWithPartial<string, RagasArgs> = makePartial(
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "extract_statements",
-            parameters: zodToJsonSchema(contextRecallSchema),
-          },
-        },
+        zodFunction({
+          name: "extract_statements",
+          parameters: contextRecallSchema,
+        }),
       ],
       tool_choice: {
         type: "function",
@@ -473,15 +464,11 @@ export const ContextPrecision: ScorerWithPartial<string, RagasArgs> =
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "verify",
-            description:
-              "Verify if context was useful in arriving at the answer",
-            parameters: zodToJsonSchema(contextPrecisionSchema),
-          },
-        },
+        zodFunction({
+          name: "verify",
+          description: "Verify if context was useful in arriving at the answer",
+          parameters: contextPrecisionSchema,
+        }),
       ],
       tool_choice: { type: "function", function: { name: "verify" } },
     });
@@ -600,14 +587,11 @@ export const Faithfulness: ScorerWithPartial<string, RagasArgs> = makePartial(
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "extract_statements",
-            description: "Extract statements from an answer given a question",
-            parameters: zodToJsonSchema(extractedStatementsSchema),
-          },
-        },
+        zodFunction({
+          name: "extract_statements",
+          description: "Extract statements from an answer given a question",
+          parameters: extractedStatementsSchema,
+        }),
       ],
       tool_choice: {
         type: "function",
@@ -631,15 +615,12 @@ export const Faithfulness: ScorerWithPartial<string, RagasArgs> = makePartial(
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "judge_statements",
-            description:
-              "Judge whether the statements are faithful to the context",
-            parameters: zodToJsonSchema(statementFaithfulnessSchema),
-          },
-        },
+        zodFunction({
+          name: "judge_statements",
+          description:
+            "Judge whether the statements are faithful to the context",
+          parameters: statementFaithfulnessSchema,
+        }),
       ],
       tool_choice: { type: "function", function: { name: "judge_statements" } },
     });
@@ -741,15 +722,12 @@ export const AnswerRelevancy: ScorerWithPartial<
           },
         ],
         tools: [
-          {
-            type: "function",
-            function: {
-              name: "generate_question",
-              description:
-                "Generate a question for the given answer and identify if the answer is noncommittal",
-              parameters: zodToJsonSchema(questionGenSchema),
-            },
-          },
+          zodFunction({
+            name: "generate_question",
+            description:
+              "Generate a question for the given answer and identify if the answer is noncommittal",
+            parameters: questionGenSchema,
+          }),
         ],
         tool_choice: {
           type: "function",
@@ -920,14 +898,11 @@ export const AnswerCorrectness: ScorerWithPartial<
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "classify_statements",
-            description: "Classify statements as TP, FP, or FN",
-            parameters: zodToJsonSchema(answerCorrectnessClassificationSchema),
-          },
-        },
+        zodFunction({
+          name: "classify_statements",
+          description: "Classify statements as TP, FP, or FN",
+          parameters: answerCorrectnessClassificationSchema,
+        }),
       ],
       tool_choice: {
         type: "function",

js/templates.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v3";
 import * as yaml from "js-yaml";
 
 import battle from "../templates/battle.yaml";

package.json

@@ -48,7 +48,11 @@
     "typedoc": "^0.25.13",
     "typedoc-plugin-markdown": "^3.17.1",
     "typescript": "^5.9.2",
-    "vitest": "^2.1.9"
+    "vitest": "^2.1.9",
+    "zod": "3.25.67"
+  },
+  "peerDependencies": {
+    "zod": "^3.25.0 || ^4.0.0"
   },
   "dependencies": {
     "ajv": "^8.17.1",
@@ -57,9 +61,8 @@
     "js-yaml": "^4.1.0",
     "linear-sum-assignment": "^1.0.7",
     "mustache": "^4.2.0",
-    "openai": "^6.3.0",
-    "zod": "^3.25.76",
-    "zod-to-json-schema": "^3.24.6"
+    "openai": "^6.7.0",
+    "zod-to-json-schema": "3.25.0"
   },
   "packageManager": "pnpm@10.33.0"
 }