From 0c0e375ad58dfd4fae01e85292f58e2d52a72cdb Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 07:35:05 +0200 Subject: [PATCH] feat(eval): load explicit TypeScript eval configs --- README.md | 10 +- apps/cli/src/commands/eval/shared.ts | 7 +- apps/cli/src/commands/validate/index.ts | 2 +- .../src/commands/validate/validate-files.ts | 33 ++-- apps/cli/test/commands/eval/discover.test.ts | 36 ++++ apps/cli/test/commands/eval/shared.test.ts | 49 ++++++ .../commands/validate/validate-files.test.ts | 114 ++++++++++++ .../docs/docs/next/evaluation/eval-files.mdx | 34 +++- .../content/docs/docs/next/evaluation/sdk.mdx | 53 +++--- .../docs/next/reference/promptfoo-parity.mdx | 1 + .../content/docs/docs/next/tools/validate.mdx | 10 +- examples/README.md | 5 +- examples/features/README.md | 2 +- .../features/sdk-eval-authoring/README.md | 12 +- .../sdk-eval-authoring/evals/greeting.eval.ts | 17 +- .../fixtures/shared-context.md | 1 - .../src/evaluation/loaders/config-loader.ts | 1 + .../src/evaluation/loaders/ts-eval-loader.ts | 162 ++++++++++++++---- .../evaluation/validation/eval-validator.ts | 30 ++++ .../core/src/evaluation/validation/index.ts | 3 +- packages/core/src/index.ts | 2 + .../.agentv/assertions/custom-check.ts | 3 + .../loaders/fixtures/invalid-contract.eval.ts | 5 + .../loaders/fixtures/module.eval.mts | 14 ++ .../loaders/fixtures/relative-import.eval.ts | 22 +++ .../loaders/fixtures/relative-prompt.ts | 1 + .../evaluation/loaders/ts-eval-loader.test.ts | 62 +++++-- packages/sdk/README.md | 35 ++-- packages/sdk/src/eval.ts | 27 +-- packages/sdk/src/index.ts | 4 +- packages/sdk/test/eval-authoring.test.ts | 4 +- packages/sdk/test/package-graph.test.ts | 12 ++ skills-data/agentv-eval-writer/SKILL.md | 16 +- .../references/custom-evaluators.md | 15 +- 34 files changed, 643 insertions(+), 161 deletions(-) create mode 100644 apps/cli/test/commands/eval/discover.test.ts create mode 100644 apps/cli/test/commands/validate/validate-files.test.ts delete mode 100644 examples/features/sdk-eval-authoring/fixtures/shared-context.md create mode 100644 packages/core/test/evaluation/loaders/fixtures/.agentv/assertions/custom-check.ts create mode 100644 packages/core/test/evaluation/loaders/fixtures/invalid-contract.eval.ts create mode 100644 packages/core/test/evaluation/loaders/fixtures/module.eval.mts create mode 100644 packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts create mode 100644 packages/core/test/evaluation/loaders/fixtures/relative-prompt.ts diff --git a/README.md b/README.md index 806c38cb5..97779b7f8 100644 --- a/README.md +++ b/README.md @@ -240,12 +240,12 @@ const { results, summary } = await evaluate({ console.log(`${summary.passed}/${summary.total} passed`); ``` -Use `defineEval()` when you want AgentV to run the TypeScript eval file: +Use `*.eval.ts` when you want AgentV to run a TypeScript eval config: ```typescript -import { defineEval } from '@agentv/sdk'; +import type { EvalConfig } from '@agentv/sdk'; -export default defineEval({ +const config: EvalConfig = { description: 'Code generation quality', tags: { experiment: 'with-skills' }, target: { @@ -285,7 +285,9 @@ export default defineEval({ ], }, ], -}); +}; + +export default config; ``` ## Documentation diff --git a/apps/cli/src/commands/eval/shared.ts b/apps/cli/src/commands/eval/shared.ts index e7bc79510..9c3fa6a39 100644 --- a/apps/cli/src/commands/eval/shared.ts +++ b/apps/cli/src/commands/eval/shared.ts @@ -1,6 +1,7 @@ import { constants } from 'node:fs'; import { access, stat } from 'node:fs/promises'; import path from 'node:path'; +import { isTypeScriptEvalConfigFileName, typeScriptEvalConfigGlob } from '@agentv/core'; import fg from 'fast-glob'; import { isAgentSkillsEvalsJsonFile } from '../read-adapters/agent-skills-evals.js'; @@ -10,7 +11,7 @@ export interface ResolveEvalPathOptions { } function isNativeEvalFile(filePath: string): boolean { - return /\.(ya?ml|jsonl|[cm]?ts)$/i.test(filePath); + return /\.(ya?ml|jsonl)$/i.test(filePath) || isTypeScriptEvalConfigFileName(filePath); } function shouldInspectJsonPath(filePath: string): boolean { @@ -79,8 +80,8 @@ export async function resolveEvalPaths( if (candidateStats.isDirectory()) { // Auto-expand directory to recursive eval file glob const filePattern = options.allowReadAdapters - ? '{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts,evals.json,*.evals.json}' - : '{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts}'; + ? `{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,${typeScriptEvalConfigGlob()},evals.json,*.evals.json}` + : `{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,${typeScriptEvalConfigGlob()}}`; const dirGlob = path.posix.join(candidatePath.replace(/\\/g, '/'), `**/${filePattern}`); const dirMatches = await fg(dirGlob, { absolute: true, diff --git a/apps/cli/src/commands/validate/index.ts b/apps/cli/src/commands/validate/index.ts index 34b487198..a88a1f8ee 100644 --- a/apps/cli/src/commands/validate/index.ts +++ b/apps/cli/src/commands/validate/index.ts @@ -40,7 +40,7 @@ async function runValidateCommand( export const validateCommand = command({ name: 'validate', - description: 'Validate AgentV eval and targets YAML files', + description: 'Validate AgentV eval, TypeScript config, and targets files', args: { paths: restPositionals({ type: string, diff --git a/apps/cli/src/commands/validate/validate-files.ts b/apps/cli/src/commands/validate/validate-files.ts index ef00c91e9..24715faf6 100644 --- a/apps/cli/src/commands/validate/validate-files.ts +++ b/apps/cli/src/commands/validate/validate-files.ts @@ -5,11 +5,13 @@ import { type ValidationResult, type ValidationSummary, detectFileType, + isTypeScriptEvalConfigFileName, validateCasesFile, validateConfigFile, validateEvalFile, validateFileReferences, validateTargetsFile, + validateTypeScriptEvalConfigFile, validateWorkspacePaths, } from '@agentv/core/evaluation/validation'; import fg from 'fast-glob'; @@ -35,6 +37,10 @@ export async function validateFiles(paths: readonly string[]): Promise { const absolutePath = path.resolve(filePath); + if (isTypeScriptEvalConfigFileName(absolutePath)) { + return validateTypeScriptEvalConfigFile(absolutePath); + } + // Detect file type (now infers from path if $schema is missing) const fileType = await detectFileType(absolutePath); @@ -97,12 +103,14 @@ async function expandPaths(paths: readonly string[]): Promise const stats = await stat(absolutePath); if (stats.isFile()) { - if (isYamlFile(absolutePath)) expanded.add(absolutePath); + if (isYamlFile(absolutePath) || isTypeScriptEvalConfigFileName(absolutePath)) { + expanded.add(absolutePath); + } continue; } if (stats.isDirectory()) { - const yamlFiles = await findYamlFiles(absolutePath); - for (const f of yamlFiles) expanded.add(f); + const evalFiles = await findEvalFiles(absolutePath); + for (const f of evalFiles) expanded.add(f); continue; } } catch { @@ -120,11 +128,13 @@ async function expandPaths(paths: readonly string[]): Promise followSymbolicLinks: true, }); - const yamlMatches = matches.filter((f) => isYamlFile(f)); - if (yamlMatches.length === 0) { - console.warn(`Warning: No YAML files matched pattern: ${inputPath}`); + const evalMatches = matches.filter((f) => isYamlFile(f) || isTypeScriptEvalConfigFileName(f)); + if (evalMatches.length === 0) { + console.warn( + `Warning: No eval YAML or TypeScript config files matched pattern: ${inputPath}`, + ); } - for (const f of yamlMatches) expanded.add(path.normalize(f)); + for (const f of evalMatches) expanded.add(path.normalize(f)); } const sorted = Array.from(expanded); @@ -132,7 +142,7 @@ async function expandPaths(paths: readonly string[]): Promise return sorted; } -async function findYamlFiles(dirPath: string): Promise { +async function findEvalFiles(dirPath: string): Promise { const results: string[] = []; try { @@ -146,9 +156,12 @@ async function findYamlFiles(dirPath: string): Promise { if (entry.name === 'node_modules' || entry.name.startsWith('.')) { continue; } - const subFiles = await findYamlFiles(fullPath); + const subFiles = await findEvalFiles(fullPath); results.push(...subFiles); - } else if (entry.isFile() && isEvalYamlFile(entry.name)) { + } else if ( + entry.isFile() && + (isEvalYamlFile(entry.name) || isTypeScriptEvalConfigFileName(entry.name)) + ) { results.push(fullPath); } } diff --git a/apps/cli/test/commands/eval/discover.test.ts b/apps/cli/test/commands/eval/discover.test.ts new file mode 100644 index 000000000..0e5804965 --- /dev/null +++ b/apps/cli/test/commands/eval/discover.test.ts @@ -0,0 +1,36 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { discoverEvalFiles } from '../../../src/commands/eval/discover.js'; + +describe('discoverEvalFiles', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-discover-eval-files-')); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('discovers explicit TypeScript eval config files without arbitrary TypeScript configs', async () => { + const evalDir = path.join(tempDir, 'evals'); + mkdirSync(evalDir, { recursive: true }); + + const tsFile = path.join(evalDir, 'greeting.eval.ts'); + const mtsFile = path.join(evalDir, 'module.eval.mts'); + writeFileSync(tsFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n'); + writeFileSync(mtsFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n'); + writeFileSync(path.join(evalDir, 'helper.ts'), 'export const helper = true;\n'); + writeFileSync(path.join(evalDir, 'agentvconfig.ts'), 'export default { tests: [] };\n'); + writeFileSync(path.join(evalDir, 'promptfooconfig.ts'), 'export default { tests: [] };\n'); + + const discovered = await discoverEvalFiles(tempDir); + const relativePaths = discovered.map((file) => file.relativePath); + + expect(relativePaths).toEqual(['evals/greeting.eval.ts', 'evals/module.eval.mts']); + }); +}); diff --git a/apps/cli/test/commands/eval/shared.test.ts b/apps/cli/test/commands/eval/shared.test.ts index 9734912c6..92e3538b3 100644 --- a/apps/cli/test/commands/eval/shared.test.ts +++ b/apps/cli/test/commands/eval/shared.test.ts @@ -108,6 +108,32 @@ describe('resolveEvalPaths', () => { expect(resolved).toEqual([path.normalize(tsFile)]); }); + it('does not discover agentvconfig.ts files from directory auto-expansion', async () => { + const evalDir = path.join(tempDir, 'evals'); + mkdirSync(evalDir, { recursive: true }); + + const tsFile = path.join(evalDir, 'agentvconfig.ts'); + const helperFile = path.join(evalDir, 'helper.ts'); + writeFileSync(tsFile, 'export default { tests: [] }'); + writeFileSync(helperFile, 'export const helper = true'); + + await expect(resolveEvalPaths([tempDir], tempDir)).rejects.toThrow( + 'No eval files matched any provided paths or globs', + ); + }); + + it('does not discover promptfooconfig.ts files from directory auto-expansion', async () => { + const evalDir = path.join(tempDir, 'evals'); + mkdirSync(evalDir, { recursive: true }); + + const tsFile = path.join(evalDir, 'promptfooconfig.ts'); + writeFileSync(tsFile, 'export default { tests: [] }'); + + await expect(resolveEvalPaths([tempDir], tempDir)).rejects.toThrow( + 'No eval files matched any provided paths or globs', + ); + }); + it('discovers Agent Skills evals.json from directory auto-expansion when read adapters are enabled', async () => { const evalDir = path.join(tempDir, 'skills', 'demo', 'evals'); mkdirSync(evalDir, { recursive: true }); @@ -144,6 +170,29 @@ describe('resolveEvalPaths', () => { expect(resolved).toEqual([path.normalize(tsFile)]); }); + it('does not accept arbitrary direct .ts file paths as eval configs', async () => { + const tsFile = path.join(tempDir, 'helper.ts'); + writeFileSync(tsFile, 'export const helper = true'); + + await expect(resolveEvalPaths([tsFile], tempDir)).rejects.toThrow( + 'No eval files matched any provided paths or globs', + ); + }); + + it('filters arbitrary TypeScript files from broad globs', async () => { + const evalDir = path.join(tempDir, 'evals'); + mkdirSync(evalDir, { recursive: true }); + + const tsFile = path.join(evalDir, 'custom.eval.mts'); + const helperFile = path.join(evalDir, 'helper.ts'); + writeFileSync(tsFile, 'export default { tests: [] }'); + writeFileSync(helperFile, 'export const helper = true'); + + const resolved = await resolveEvalPaths(['evals/**/*.ts', 'evals/**/*.mts'], tempDir); + + expect(resolved).toEqual([path.normalize(tsFile)]); + }); + it('discovers both .yaml and .ts files from directory', async () => { const evalDir = path.join(tempDir, 'evals'); mkdirSync(evalDir, { recursive: true }); diff --git a/apps/cli/test/commands/validate/validate-files.test.ts b/apps/cli/test/commands/validate/validate-files.test.ts new file mode 100644 index 000000000..b53c4acab --- /dev/null +++ b/apps/cli/test/commands/validate/validate-files.test.ts @@ -0,0 +1,114 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { validateFiles } from '../../../src/commands/validate/validate-files.js'; + +describe('validateFiles TypeScript eval configs', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-validate-ts-config-')); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('validates *.eval.ts through the core loader', async () => { + const configFile = path.join(tempDir, 'suite.eval.ts'); + writeFileSync( + configFile, + `export default { + prompts: ['{{ input }}'], + tests: [ + { + id: 'hello', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], +}; +`, + ); + + const summary = await validateFiles([configFile]); + + expect(summary.totalFiles).toBe(1); + expect(summary.validFiles).toBe(1); + expect(summary.results[0].filePath).toBe(path.normalize(configFile)); + }); + + it('reports missing default export errors for TypeScript configs', async () => { + const configFile = path.join(tempDir, 'suite.eval.ts'); + writeFileSync(configFile, 'export const config = { tests: [] };\n'); + + const summary = await validateFiles([configFile]); + + expect(summary.invalidFiles).toBe(1); + expect(summary.results[0].errors[0].message).toContain('default export'); + }); + + it('reports invalid TypeScript eval contract errors', async () => { + const configFile = path.join(tempDir, 'suite.eval.mts'); + writeFileSync( + configFile, + `export default { + prompts: ['{{ input }}'], + providers: ['openai:gpt-5'], + tests: [{ id: 'hello', vars: { input: 'Say hello' } }], +}; +`, + ); + + const summary = await validateFiles([configFile]); + + expect(summary.invalidFiles).toBe(1); + expect(summary.results[0].errors[0].message).toContain("top-level 'providers'"); + }); + + it('expands directories without treating custom assertion files as eval configs', async () => { + const assertionsDir = path.join(tempDir, '.agentv', 'assertions'); + mkdirSync(assertionsDir, { recursive: true }); + writeFileSync(path.join(assertionsDir, 'custom-check.ts'), 'export default () => true;\n'); + const configFile = path.join(tempDir, 'suite.eval.ts'); + writeFileSync( + configFile, + `export default { + prompts: ['{{ input }}'], + tests: [ + { + id: 'hello', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], +}; +`, + ); + + const summary = await validateFiles([tempDir]); + + expect(summary.totalFiles).toBe(1); + expect(summary.results[0].filePath).toBe(path.normalize(configFile)); + }); + + it('does not validate promptfooconfig.ts as an eval config', async () => { + const configFile = path.join(tempDir, 'promptfooconfig.ts'); + writeFileSync(configFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n'); + + const summary = await validateFiles([tempDir]); + + expect(summary.totalFiles).toBe(0); + }); + + it('does not validate agentvconfig.ts as an eval config', async () => { + const configFile = path.join(tempDir, 'agentvconfig.ts'); + writeFileSync(configFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n'); + + const summary = await validateFiles([tempDir]); + + expect(summary.totalFiles).toBe(0); + }); +}); diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index ff94fd061..202872f19 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -15,10 +15,12 @@ attempts and gates. Coding-agent testbeds, workdirs, Docker config, repository materialization, setup, and reset policy belong in `environment`. Provider environment-variable overrides belong in top-level `env`. Lifecycle hooks belong in `extensions`; runner-specific setup belongs in the `target` object, in -`targets`, or in project config. AgentV supports two eval data formats: YAML and -JSONL. +`targets`, or in project config. AgentV supports YAML, JSONL, and explicit +TypeScript eval config files. -YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. +YAML is the canonical portable model. TypeScript helpers, generated fixtures, +and Python scripts should lower to the same YAML/JSONL shapes rather than +inventing a separate eval contract. Eval files describe the task, target binding, and run controls. Use `evaluate_options.max_concurrency` for authored suite concurrency. Operators can still override concurrency with `agentv eval --workers N`; do not author legacy @@ -37,6 +39,32 @@ focused, shareable slice of the same config graph as `.agentv/config.yaml`. Use ordinary `*.eval.yaml` files for direct task suites. Raw case files are reusable data inputs, not a second runnable experiment format. +For TypeScript config authoring, use an explicit `*.eval.ts` or `*.eval.mts` +file with a typed default export: + +```typescript +import type { EvalConfig } from '@agentv/sdk'; + +const config: EvalConfig = { + target: 'local-mini', + prompts: ['Summarize {{ topic }} for {{ audience }}.'], + tests: [ + { + id: 'release-notes', + vars: { topic: 'the July release notes', audience: 'engineers' }, + assert: [{ type: 'contains', value: 'July' }], + }, + ], +}; + +export default config; +``` + +Directory discovery includes `*.eval.ts` and `*.eval.mts`. Arbitrary helper +`.ts` files are not eval configs. Custom assertions remain under +`.agentv/assertions/*.ts` and are loaded through assertion discovery, not +through the eval config loader. + - A **task suite** is eval YAML that owns task context: `environment`, shared `prompts`, shared `assert`, fixtures, graders, and test cases. It can run directly. diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index e7a721deb..bb7114fe1 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -43,7 +43,7 @@ The general policy is hard convergence for same-week or unreleased surface names Use the simplest surface that matches the job: - **YAML / JSONL first** for portable eval specs you want to run from the CLI, check into a repo, or share across TypeScript and Python workflows. -- **`defineEval()` / `evalSuite()`** when you want a `.eval.ts` file that mirrors YAML concepts and lowers back to the canonical snake_case contract. +- **`EvalConfig` default exports** when you want a TypeScript `*.eval.ts` file that mirrors YAML concepts and lowers back to the canonical snake_case contract. - **`evaluate({ specFile })`** when you want library control around an existing YAML suite. - **Inline `evaluate({ tests })`** when the eval definition truly belongs inside application code. The programmatic API mirrors YAML, but uses current TypeScript naming such as `expectedOutput`. - **`defineAssertion`** when you want a reusable assertion type discovered from `.agentv/assertions/`. @@ -87,56 +87,53 @@ write_eval_yaml( This is example-local/repo-local guidance, not a promise of a published Python package. -## YAML-Aligned `.eval.ts` Authoring +## TypeScript Eval Config Authoring -Use `defineEval()` from `@agentv/sdk` when you want TypeScript ergonomics without creating a second eval vocabulary. The helper keeps authoring in camelCase where TypeScript needs it, then lowers back to the canonical snake_case eval object contract when AgentV loads the file. +Use `EvalConfig` from `@agentv/sdk` when you want TypeScript ergonomics without creating a second eval vocabulary. AgentV accepts explicit `*.eval.ts` and `*.eval.mts` files as eval configs. Each TypeScript config uses a default export and lowers back to the canonical snake_case eval object contract when AgentV loads the file. ```typescript // evals/greeting.eval.ts -import { defineEval, graders } from '@agentv/sdk'; +import { graders, type EvalConfig } from '@agentv/sdk'; -export default defineEval({ +const config: EvalConfig = { name: 'hello-suite', target: 'mock-sdk', - workspace: { - hooks: { - beforeAll: { - command: ['echo', 'suite-start'], - }, - }, - }, + prompts: ['{{ task }}'], tests: [ { id: 'hello', - input: 'Say hello', + vars: { task: 'Say hello' }, inputFiles: ['../fixtures/per-test-note.md'], - expectedOutput: 'Hello from the mock target', assert: [graders.contains('Hello')], }, ], -}); +}; + +export default config; ``` Useful companion helpers: - `toEvalYamlObject()` returns the canonical snake_case object. - `serializeEvalYaml()` returns YAML text using the same canonical field names. +- `defineEval(config)` is a thin optional helper over the same `EvalConfig` shape. -The durable authored field remains `assert`. This helper does not introduce a second YAML vocabulary. +The durable authored field remains `assert`. TypeScript eval config authoring does not introduce a second YAML vocabulary. ## Built-In Grader Helpers `@agentv/sdk` includes a small `graders` catalog for common deterministic and LLM-backed grader configs. These helpers return ordinary `assert` entries and serialize to the same canonical YAML you could write by hand. ```typescript -import { defineEval, graders } from '@agentv/sdk'; +import { graders, type EvalConfig } from '@agentv/sdk'; -export default defineEval({ +const config: EvalConfig = { name: 'grader-helper-suite', + prompts: ['{{ input }}'], tests: [ { id: 'json-greeting', - input: 'Return a JSON greeting.', + vars: { input: 'Return a JSON greeting.' }, assert: [ graders.contains('Hello', { name: 'mentions-hello' }), graders.exact('{"message":"Hello"}', { name: 'exact-json', minScore: 1 }), @@ -152,17 +149,19 @@ export default defineEval({ ], }, ], -}); +}; + +export default config; ``` The catalog covers `contains`, `equals`/`exact`, `regex`, `is-json`/`json`, `llm-rubric`, and `script`. CamelCase SDK options such as `minScore`, `maxSteps`, and rubric `scoreRanges` lower to `min_score`, `max_steps`, and `score_ranges` when AgentV loads or serializes the suite. ## AgentV-Native Helper Factories -If you are coming from Braintrust `scores` or DeepEval metrics, keep the reusable logic AgentV-native: write helper factories that return `graders.*` configs, then let `defineEval()` lower them to ordinary `assert` entries. +If you are coming from Braintrust `scores` or DeepEval metrics, keep the reusable logic AgentV-native: write helper factories that return `graders.*` configs, then let the TypeScript eval config lower them to ordinary `assert` entries. ```typescript -import { defineEval, graders } from '@agentv/sdk'; +import { graders, type EvalConfig } from '@agentv/sdk'; function ragFaithfulness() { return graders.llmRubric(undefined, { @@ -175,20 +174,22 @@ function ragFaithfulness() { }); } -export default defineEval({ +const config: EvalConfig = { name: 'rag-suite', + prompts: ['{{ input }}'], tests: [ { id: 'grounded-answer', - input: 'Answer the question using the retrieved context.', - expectedOutput: 'The answer cites the source material.', + vars: { input: 'Answer the question using the retrieved context.' }, assert: [ graders.contains('source', { name: 'mentions-source' }), ragFaithfulness(), ], }, ], -}); +}; + +export default config; ``` The helper above serializes to the same shape you could write by hand: diff --git a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx index d004f34da..c47dbf0dc 100644 --- a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx +++ b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx @@ -32,6 +32,7 @@ implements equivalent semantics directly. | Surface | Promptfoo shape | AgentV shape | Decision | Notes | | --- | --- | --- | --- | --- | +| TypeScript config file | `promptfooconfig.ts` default export typed as Promptfoo `UnifiedConfig`. | Explicit `*.eval.ts` or `*.eval.mts` default export typed as `EvalConfig` from `@agentv/sdk`. | Keep AgentV divergence | AgentV does not accept `promptfooconfig.ts` by default because the config semantics intentionally diverge: top-level `providers` is not an AgentV runtime alias, and AgentV keeps `target`/`targets`, `environment`, `evaluate_options`, and `assert` as its supported contract. Directory discovery includes explicit `*.eval.ts` and `*.eval.mts` config files, but not arbitrary helper `.ts` files. | | Prompt matrix | Top-level `prompts` rendered with each test's `vars`. | Top-level `prompts` rendered with `tests[].vars` and `default_test.vars`. | Align with Promptfoo | This is the canonical Promptfoo-compatible input shape in AgentV. Prompt entries can be inline strings, chat arrays, files, or generated prompt functions. | | Test rows | `tests` can be inline rows or a case-file reference; rows carry `vars`, `assert`, metadata, prompt/provider filters, and expected data. | `tests` can be inline rows or a raw-case path; rows carry `vars`, `assert`, `expected_output`, metadata, optional environment overrides, and run overrides. | Align with Promptfoo | AgentV uses field-local file refs such as `tests: file://...`, `prompts: file://...`, and `default_test: file://...`; coding-agent testbeds use `environment: file://...`. There is no separate imports table. | | Variables | `tests[].vars` plus `defaultTest.vars`; prompt templates can reference top-level var names. | `tests[].vars` plus `default_test.vars`; templates can use `{{ name }}` or `{{ vars.name }}`. | Align with Promptfoo | Per-test vars override default vars by key. | diff --git a/apps/web/src/content/docs/docs/next/tools/validate.mdx b/apps/web/src/content/docs/docs/next/tools/validate.mdx index bd60b6801..3ac9703f0 100644 --- a/apps/web/src/content/docs/docs/next/tools/validate.mdx +++ b/apps/web/src/content/docs/docs/next/tools/validate.mdx @@ -12,22 +12,28 @@ The `validate` command checks evaluation files for schema errors without running ```bash agentv validate evals/my-eval.yaml +agentv validate evals/greeting.eval.ts ``` Validate multiple files: ```bash -agentv validate evals/**/*.yaml +agentv validate "evals/**/*.yaml" "evals/**/*.ts" ``` ## What It Checks -- YAML/JSONL syntax +- YAML/JSONL syntax and TypeScript config default exports - Required fields: `id`, input data (`input` or prompt matrix `vars`), and at least one grader under `assert` - Grader references (command paths, prompt files) - Target references match entries in `targets.yaml` - Rubric structure and field types +For TypeScript eval configs, `agentv validate` imports and materializes the file +through the same core loader used by `agentv eval`. Supported TypeScript config +filenames are `*.eval.ts` and `*.eval.mts`; arbitrary `.ts` helper files are +ignored during directory and glob expansion. + ## When to Use - Before running evaluations to catch config errors early diff --git a/examples/README.md b/examples/README.md index e2222d6de..cc0bb8a3b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -62,7 +62,7 @@ Focused demonstrations of specific AgentV capabilities. Each example includes it - [vitest-workspace-grader](features/vitest-workspace-grader/) - Built-in AgentV adapter for Vitest workspace verifier files - [sdk-custom-assertion](features/sdk-custom-assertion/) - Custom assertion types using `defineAssertion()` - [sdk-programmatic-api](features/sdk-programmatic-api/) - Programmatic evaluation using `evaluate()` -- [sdk-eval-authoring](features/sdk-eval-authoring/) - YAML-aligned `.eval.ts` authoring using `defineEval()` +- [sdk-eval-authoring](features/sdk-eval-authoring/) - TypeScript `*.eval.ts` authoring using `EvalConfig` - [sdk-config-file](features/sdk-config-file/) - Typed configuration with `defineConfig()` - [prompt-template-sdk](features/prompt-template-sdk/) - Custom LLM grader prompts using `definePromptTemplate()` @@ -87,7 +87,8 @@ Each example follows this structure: example-name/ ├── evals/ │ ├── suite.yaml # Primary eval file -│ ├── *.ts or *.py # script graders (optional) +│ ├── *.eval.ts # TypeScript eval config (optional) +│ ├── *.ts or *.py # script graders and helper code (optional) │ └── *.md # LLM grader prompts (optional) ├── scripts/ # Helper scripts (optional) ├── .agentv/ diff --git a/examples/features/README.md b/examples/features/README.md index 0aef11a71..c4999bdf3 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -121,7 +121,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | Example | Description | |---------|-------------| | [sdk-custom-assertion](sdk-custom-assertion/) | Custom assertion types using `defineAssertion()` | -| [sdk-eval-authoring](sdk-eval-authoring/) | YAML-aligned `.eval.ts` authoring using `defineEval()` | +| [sdk-eval-authoring](sdk-eval-authoring/) | TypeScript `*.eval.ts` authoring using `EvalConfig` | | [sdk-programmatic-api](sdk-programmatic-api/) | Programmatic evaluation using `evaluate()` | | [sdk-config-file](sdk-config-file/) | Typed configuration with `defineConfig()` | | [prompt-template-sdk](prompt-template-sdk/) | Custom LLM grader prompts using `definePromptTemplate()` | diff --git a/examples/features/sdk-eval-authoring/README.md b/examples/features/sdk-eval-authoring/README.md index 1571d4c21..36ff3bbeb 100644 --- a/examples/features/sdk-eval-authoring/README.md +++ b/examples/features/sdk-eval-authoring/README.md @@ -1,19 +1,19 @@ -# SDK Example: YAML-Aligned Eval Authoring +# SDK Example: TypeScript Eval Config Authoring -Demonstrates authoring a `.eval.ts` suite with `defineEval()` from `@agentv/sdk` while still lowering to AgentV's canonical snake_case YAML/runtime contract. +Demonstrates authoring an explicit `*.eval.ts` file with the `EvalConfig` type from `@agentv/sdk`. ## What It Shows -1. `defineEval()` brands a TypeScript suite for the `.eval.ts` loader. +1. `evals/greeting.eval.ts` uses a default export as the supported TypeScript eval config contract. 2. The `graders` helper catalog returns ordinary `assert` entries. -3. CamelCase authoring fields such as `inputFiles`, `expectedOutput`, `beforeAll`, and `beforeEach` lower to the canonical YAML/runtime keys. +3. CamelCase authoring fields such as per-test `inputFiles` lower to the canonical YAML/runtime keys. 4. The suite still runs through the standard CLI and YAML parser path instead of a separate SDK runner. ## Files -- `evals/greeting.eval.ts` — the YAML-aligned TypeScript suite +- `evals/greeting.eval.ts` — the TypeScript eval config - `.agentv/targets.yaml` — local mock target for a zero-credential run -- `fixtures/*.md` — attached input files used by the suite +- `fixtures/per-test-note.md` — attached input file used by the suite ## How to Run diff --git a/examples/features/sdk-eval-authoring/evals/greeting.eval.ts b/examples/features/sdk-eval-authoring/evals/greeting.eval.ts index 86af6a410..97caa675c 100644 --- a/examples/features/sdk-eval-authoring/evals/greeting.eval.ts +++ b/examples/features/sdk-eval-authoring/evals/greeting.eval.ts @@ -1,24 +1,27 @@ -import { defineEval, graders } from '@agentv/sdk'; +import { type EvalConfig, graders } from '@agentv/sdk'; -export default defineEval({ +const config: EvalConfig = { name: 'sdk-eval-authoring', - description: 'YAML-aligned TypeScript eval authoring with @agentv/sdk', - inputFiles: ['../fixtures/shared-context.md'], + description: 'TypeScript eval config authoring with @agentv/sdk', target: 'mock-sdk', environment: { type: 'host', workdir: '../fixtures', }, + prompts: ['Use the attached notes and {{ task }}'], tests: [ { id: 'hello-from-typescript', - input: 'Use the attached notes and say hello.', + vars: { + task: 'say hello.', + }, inputFiles: ['../fixtures/per-test-note.md'], - expectedOutput: 'Hello from the mock target', assert: [ graders.contains('Hello', { name: 'mentions-hello' }), graders.regex(/mock target/i, { name: 'mentions-mock-target' }), ], }, ], -}); +}; + +export default config; diff --git a/examples/features/sdk-eval-authoring/fixtures/shared-context.md b/examples/features/sdk-eval-authoring/fixtures/shared-context.md deleted file mode 100644 index d77029741..000000000 --- a/examples/features/sdk-eval-authoring/fixtures/shared-context.md +++ /dev/null @@ -1 +0,0 @@ -Use a friendly tone. diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 58279ef4f..3803af8a2 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -39,6 +39,7 @@ export const DEFAULT_EVAL_PATTERNS: readonly string[] = [ '**/evals/**/*.eval.yaml', '**/evals/**/eval.yaml', '**/evals/**/*.eval.ts', + '**/evals/**/*.eval.mts', ]; export type ExecutionDefaults = { diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index fd087fa49..1743e139c 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -1,25 +1,56 @@ /** - * Loads an eval suite from a TypeScript *.eval.ts file. + * Loads an eval suite from a TypeScript eval config file. * - * Each TS eval file must export an EvalConfig as its default export or - * as a named export called `config` or `evalConfig`. + * Each TS eval file must export an EvalConfig as its default export. + * Supported filenames are explicit AgentV *.eval.ts / *.eval.mts files. * * The file is loaded via dynamic import() which works natively in Bun * and requires tsx/jiti for Node.js. - * - * To add a new export convention: add the name to EXPORT_NAMES below. */ import path from 'node:path'; import { pathToFileURL } from 'node:url'; -import { type EvalConfig, materializeEvalConfig } from '../evaluate.js'; +import { type EvalConfig as ProgrammaticEvalConfig, materializeEvalConfig } from '../evaluate.js'; import { createFunctionProvider } from '../providers/function-provider.js'; import type { ProviderFactoryFn } from '../providers/provider-registry.js'; import type { TargetDefinition } from '../providers/types.js'; import { type EvalSuiteResult, loadTestSuiteFromYamlObject } from '../yaml-parser.js'; -const EXPORT_NAMES = ['default', 'config', 'evalConfig'] as const; const SDK_EVAL_SUITE_SYMBOL = Symbol.for('@agentv/sdk/eval-suite'); const SDK_TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for('@agentv/sdk/to-eval-yaml-object'); +const TS_EVAL_CONFIG_NAME_RE = /^.+\.eval\.(?:m)?ts$/i; +const TS_EVAL_CONFIG_GLOB = '*.eval.ts,*.eval.mts' as const; + +const KNOWN_SNAKE_CASE_KEYS = { + afterAll: 'after_all', + afterEach: 'after_each', + argsMatch: 'args_match', + beforeAll: 'before_all', + beforeEach: 'before_each', + cachePath: 'cache_path', + budgetUsd: 'budget_usd', + conversationId: 'conversation_id', + costLimitUsd: 'cost_limit_usd', + dependsOn: 'depends_on', + earlyExit: 'early_exit', + expectedOutput: 'expected_output', + failOnError: 'fail_on_error', + inputFiles: 'input_files', + keepWorkspaces: 'keep_workspaces', + maxConcurrency: 'max_concurrency', + maxCostUsd: 'max_cost_usd', + maxDurationMs: 'max_duration_ms', + maxToolCalls: 'max_tool_calls', + onDependencyFailure: 'on_dependency_failure', + onTurnFailure: 'on_turn_failure', + outputPath: 'output_path', + readOnly: 'read_only', + reasoningEffort: 'reasoning_effort', + skipDefaults: 'skip_defaults', + timeoutMs: 'timeout_ms', + timeoutSeconds: 'timeout_seconds', + useTarget: 'use_target', + windowSize: 'window_size', +} as const; type SdkEvalSuiteExport = Record & { readonly [SDK_EVAL_SUITE_SYMBOL]: true; @@ -27,7 +58,7 @@ type SdkEvalSuiteExport = Record & { }; export interface TsEvalResult { - readonly config: EvalConfig | SdkEvalSuiteExport; + readonly config: ProgrammaticEvalConfig | SdkEvalSuiteExport | Record; readonly filePath: string; } @@ -36,27 +67,29 @@ export interface TsEvalSuiteResult extends EvalSuiteResult { readonly providerFactory?: ProviderFactoryFn; } -/** - * Import a *.eval.ts file and extract the EvalConfig export. - * Tries default, `config`, and `evalConfig` named exports in priority order. - */ +export function isTypeScriptEvalConfigFileName(filePath: string): boolean { + return TS_EVAL_CONFIG_NAME_RE.test(path.basename(filePath)); +} + +export function typeScriptEvalConfigGlob(): string { + return TS_EVAL_CONFIG_GLOB; +} + +/** Import a TypeScript eval config file and extract its default EvalConfig export. */ export async function loadTsEvalFile(filePath: string): Promise { const absolutePath = path.resolve(filePath); const moduleUrl = pathToFileURL(absolutePath).href; const module = await import(moduleUrl); - let config: EvalConfig | SdkEvalSuiteExport | undefined; - for (const name of EXPORT_NAMES) { - const candidate = module[name]; - if (isSupportedTsEvalExport(candidate)) { - config = candidate; - break; - } - } - + const config = module.default; if (!config) { throw new Error( - `${filePath}: no supported eval export found. Export defineEval(...) or an EvalConfig as default, 'config', or 'evalConfig'.`, + `${filePath}: no supported eval export found. Export an EvalConfig as the default export.`, + ); + } + if (!isSupportedTsEvalExport(config)) { + throw new Error( + `${filePath}: default export must be an EvalConfig object. Export a plain EvalConfig object or defineEval(config) as default.`, ); } @@ -83,7 +116,14 @@ export async function loadTsEvalSuite( ); } - const materialized = await materializeEvalConfig(config, { + if (!isProgrammaticEvalConfig(config) && isYamlAlignedEvalConfig(config)) { + return loadTestSuiteFromYamlObject(absolutePath, lowerTypeScriptEvalConfig(config), repoRoot, { + ...options, + allowInternalExpectedOutput: true, + }); + } + + const materialized = await materializeEvalConfig(config as ProgrammaticEvalConfig, { repoRoot, baseDir: path.dirname(absolutePath), filter: options?.filter, @@ -123,16 +163,74 @@ function isSdkEvalSuiteExport(value: unknown): value is SdkEvalSuiteExport { ); } -function isSupportedTsEvalExport(value: unknown): value is EvalConfig | SdkEvalSuiteExport { - return isSdkEvalSuiteExport(value) || isEvalConfigLike(value); +function isSupportedTsEvalExport( + value: unknown, +): value is ProgrammaticEvalConfig | SdkEvalSuiteExport | Record { + return ( + isSdkEvalSuiteExport(value) || (!!value && typeof value === 'object' && !Array.isArray(value)) + ); } -/** - * Duck-type check for EvalConfig-like objects. - * An EvalConfig must have at least one of: tests, specFile, or target. - */ -function isEvalConfigLike(value: unknown): value is EvalConfig { - if (!value || typeof value !== 'object') return false; +function isYamlAlignedEvalConfig(value: unknown): value is Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const obj = value as Record; + return typeof obj.task !== 'function' && obj.specFile === undefined; +} + +function isProgrammaticEvalConfig(value: unknown): value is ProgrammaticEvalConfig { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; const obj = value as Record; - return 'tests' in obj || 'specFile' in obj || 'target' in obj || 'task' in obj; + const target = obj.target; + return ( + typeof obj.task === 'function' || + typeof obj.specFile === 'string' || + (!!target && + typeof target === 'object' && + !Array.isArray(target) && + typeof (target as Record).name === 'string') + ); +} + +function lowerTypeScriptEvalConfig(config: Record): Record { + const lowered = lowerEvalYamlValue(config) as Record; + const { budget_usd: budgetUsd, repeat, ...withoutRuntimeAliases } = lowered; + if (budgetUsd === undefined && repeat === undefined) { + return lowered; + } + + const evaluateOptions = + withoutRuntimeAliases.evaluate_options && + typeof withoutRuntimeAliases.evaluate_options === 'object' && + !Array.isArray(withoutRuntimeAliases.evaluate_options) + ? { ...(withoutRuntimeAliases.evaluate_options as Record) } + : {}; + + if (budgetUsd !== undefined && evaluateOptions.budget_usd === undefined) { + evaluateOptions.budget_usd = budgetUsd; + } + if (repeat !== undefined && evaluateOptions.repeat === undefined) { + evaluateOptions.repeat = repeat; + } + + return { + ...withoutRuntimeAliases, + evaluate_options: evaluateOptions, + }; +} + +function lowerEvalYamlValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => lowerEvalYamlValue(item)); + } + + if (value && typeof value === 'object') { + const result: Record = {}; + for (const [key, nestedValue] of Object.entries(value)) { + const loweredKey = KNOWN_SNAKE_CASE_KEYS[key as keyof typeof KNOWN_SNAKE_CASE_KEYS] ?? key; + result[loweredKey] = lowerEvalYamlValue(nestedValue); + } + return result; + } + + return value; } diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index b20d7b6f8..02c82b563 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -776,6 +776,36 @@ export async function validateEvalFile(filePath: string): Promise { + const absolutePath = path.resolve(filePath); + + try { + const { loadTsEvalSuite } = await import('../loaders/ts-eval-loader.js'); + await loadTsEvalSuite(absolutePath, process.cwd()); + return { + valid: true, + filePath: absolutePath, + fileType: 'eval', + errors: [], + }; + } catch (error) { + return { + valid: false, + filePath: absolutePath, + fileType: 'eval', + errors: [ + { + severity: 'error', + filePath: absolutePath, + message: (error as Error).message, + }, + ], + }; + } +} + async function validateSuiteWorkspaceConfigs( parsed: JsonObject, absolutePath: string, diff --git a/packages/core/src/evaluation/validation/index.ts b/packages/core/src/evaluation/validation/index.ts index 23093ae0b..9e5689ba3 100644 --- a/packages/core/src/evaluation/validation/index.ts +++ b/packages/core/src/evaluation/validation/index.ts @@ -3,7 +3,8 @@ */ export { detectFileType, isValidSchema, getExpectedSchema } from './file-type.js'; -export { validateEvalFile } from './eval-validator.js'; +export { isTypeScriptEvalConfigFileName } from '../loaders/ts-eval-loader.js'; +export { validateEvalFile, validateTypeScriptEvalConfigFile } from './eval-validator.js'; export { validateCasesFile } from './cases-validator.js'; export { validateTargetsFile } from './targets-validator.js'; export { validateConfigFile } from './config-validator.js'; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b255e8631..433f47808 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -29,7 +29,9 @@ export { type EnvFromFormat, } from './evaluation/loaders/config-loader.js'; export { + isTypeScriptEvalConfigFileName, loadTsEvalFile, + typeScriptEvalConfigGlob, type TsEvalResult, } from './evaluation/loaders/ts-eval-loader.js'; export type { diff --git a/packages/core/test/evaluation/loaders/fixtures/.agentv/assertions/custom-check.ts b/packages/core/test/evaluation/loaders/fixtures/.agentv/assertions/custom-check.ts new file mode 100644 index 000000000..21c925446 --- /dev/null +++ b/packages/core/test/evaluation/loaders/fixtures/.agentv/assertions/custom-check.ts @@ -0,0 +1,3 @@ +export default function customCheck() { + return { pass: true, score: 1 }; +} diff --git a/packages/core/test/evaluation/loaders/fixtures/invalid-contract.eval.ts b/packages/core/test/evaluation/loaders/fixtures/invalid-contract.eval.ts new file mode 100644 index 000000000..920e8e678 --- /dev/null +++ b/packages/core/test/evaluation/loaders/fixtures/invalid-contract.eval.ts @@ -0,0 +1,5 @@ +export default { + prompts: ['{{ input }}'], + providers: ['openai:gpt-5'], + tests: [{ id: 'invalid', vars: { input: 'Say hello' } }], +}; diff --git a/packages/core/test/evaluation/loaders/fixtures/module.eval.mts b/packages/core/test/evaluation/loaders/fixtures/module.eval.mts new file mode 100644 index 000000000..01c9dc136 --- /dev/null +++ b/packages/core/test/evaluation/loaders/fixtures/module.eval.mts @@ -0,0 +1,14 @@ +const config = { + name: 'module-mts-config', + target: 'mock-target', + prompts: ['{{ input }}'], + tests: [ + { + id: 'module-mts-config', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], +}; + +export default config; diff --git a/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts b/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts new file mode 100644 index 000000000..403991806 --- /dev/null +++ b/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts @@ -0,0 +1,22 @@ +import { relativePrompt } from './relative-prompt.ts'; + +const config = { + name: 'relative-import-ts-config', + target: 'mock-target', + tags: { experiment: 'ts-config', group: 'loader' }, + prompts: [relativePrompt], + budgetUsd: 1, + repeat: { + count: 2, + strategy: 'pass_any', + }, + tests: [ + { + id: 'relative-import', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], +}; + +export default config; diff --git a/packages/core/test/evaluation/loaders/fixtures/relative-prompt.ts b/packages/core/test/evaluation/loaders/fixtures/relative-prompt.ts new file mode 100644 index 000000000..564f910de --- /dev/null +++ b/packages/core/test/evaluation/loaders/fixtures/relative-prompt.ts @@ -0,0 +1 @@ +export const relativePrompt = '{{ input }}'; diff --git a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts index 8b6cc418d..b481ec186 100644 --- a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts +++ b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from 'bun:test'; import path from 'node:path'; -import { loadTsEvalFile } from '../../../src/evaluation/loaders/ts-eval-loader.js'; +import { + isTypeScriptEvalConfigFileName, + loadTsEvalFile, +} from '../../../src/evaluation/loaders/ts-eval-loader.js'; import { loadTestSuite, loadTests } from '../../../src/evaluation/yaml-parser.js'; const fixtureDir = path.join(import.meta.dir, 'fixtures'); @@ -15,18 +18,13 @@ describe('loadTsEvalFile', () => { expect(tests?.[0]?.id).toBe('greeting'); }); - it('loads named "config" export', async () => { - const result = await loadTsEvalFile(path.join(fixtureDir, 'named-config.eval.ts')); - const tests = (result.config as { tests?: Array<{ id?: string }> }).tests; - expect(result.config).toBeDefined(); - expect(tests?.[0]?.id).toBe('named-config'); - }); - - it('loads named "evalConfig" export', async () => { - const result = await loadTsEvalFile(path.join(fixtureDir, 'eval-config-named.eval.ts')); - const tests = (result.config as { tests?: Array<{ id?: string }> }).tests; - expect(result.config).toBeDefined(); - expect(tests?.[0]?.id).toBe('eval-config-named'); + it('rejects named config exports', async () => { + await expect(loadTsEvalFile(path.join(fixtureDir, 'named-config.eval.ts'))).rejects.toThrow( + 'Export an EvalConfig as the default export', + ); + await expect( + loadTsEvalFile(path.join(fixtureDir, 'eval-config-named.eval.ts')), + ).rejects.toThrow('Export an EvalConfig as the default export'); }); it('loads YAML-aligned sdk eval exports', async () => { @@ -38,7 +36,7 @@ describe('loadTsEvalFile', () => { it('throws when no supported eval export is found', async () => { await expect(loadTsEvalFile(path.join(fixtureDir, 'no-config.eval.ts'))).rejects.toThrow( - 'no supported eval export found', + 'default export', ); }); @@ -63,6 +61,31 @@ describe('loadTsEvalFile', () => { expect(suite.inlineTarget?.name).toBe('inline-target'); }); + it('materializes *.eval.ts default exports with relative imports', async () => { + const suite = await loadTestSuite(path.join(fixtureDir, 'relative-import.eval.ts'), fixtureDir); + + expect(suite.tests).toHaveLength(1); + expect(suite.tests[0].id).toBe('relative-import'); + expect(suite.tests[0].input).toEqual([{ role: 'user', content: 'Say hello' }]); + expect(suite.targetSpec).toEqual({ name: 'mock-target' }); + expect(suite.budgetUsd).toBe(1); + expect(suite.experimentConfig?.repeat?.count).toBe(2); + expect(suite.tags).toEqual({ experiment: 'ts-config', group: 'loader' }); + }); + + it('materializes *.eval.mts default exports', async () => { + const suite = await loadTestSuite(path.join(fixtureDir, 'module.eval.mts'), fixtureDir); + + expect(suite.tests).toHaveLength(1); + expect(suite.tests[0].id).toBe('module-mts-config'); + }); + + it('reports invalid TypeScript eval contracts through suite loading', async () => { + await expect( + loadTestSuite(path.join(fixtureDir, 'invalid-contract.eval.ts'), fixtureDir), + ).rejects.toThrow("top-level 'providers'"); + }); + it('materializes a YAML-aligned sdk eval through loadTestSuite', async () => { const suite = await loadTestSuite( path.join(fixtureDir, 'sdk-define-eval.eval.ts'), @@ -93,4 +116,15 @@ describe('loadTsEvalFile', () => { expect(tests[0].id).toBe('greeting'); expect(tests[0].category).toBe('sdk'); }); + + it('recognizes only explicit TypeScript eval config filenames', () => { + expect(isTypeScriptEvalConfigFileName('promptfooconfig.ts')).toBe(false); + expect(isTypeScriptEvalConfigFileName('promptfooconfig.mts')).toBe(false); + expect(isTypeScriptEvalConfigFileName('agentvconfig.ts')).toBe(false); + expect(isTypeScriptEvalConfigFileName('agentvconfig.mts')).toBe(false); + expect(isTypeScriptEvalConfigFileName('suite.eval.ts')).toBe(true); + expect(isTypeScriptEvalConfigFileName('suite.eval.mts')).toBe(true); + expect(isTypeScriptEvalConfigFileName('.agentv/assertions/custom-check.ts')).toBe(false); + expect(isTypeScriptEvalConfigFileName('helpers/custom-check.ts')).toBe(false); + }); }); diff --git a/packages/sdk/README.md b/packages/sdk/README.md index c76b71556..0ea4e1823 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -156,13 +156,13 @@ export default defineWorkspaceGrader(async ({ workspace }) => [ The helper resolves `workspace_path` or `AGENTV_WORKSPACE_PATH`, reads files relative to the workspace, returns AgentV check objects, and computes `score` as passed checks divided by total checks. Prefer Vitest verifiers for checks that naturally fit a test file; use this lower-level helper for tiny one-off graders or custom score shaping. -### defineEval (YAML-aligned `.eval.ts` authoring) +### TypeScript eval config authoring ```typescript -#!/usr/bin/env bun -import { defineEval, graders } from '@agentv/sdk'; +// evals/greeting.eval.ts +import { graders, type EvalConfig } from '@agentv/sdk'; -export default defineEval({ +const config: EvalConfig = { name: 'hello-suite', target: 'mock-sdk', prompts: ['{{ input }}'], @@ -174,19 +174,21 @@ export default defineEval({ assert: [graders.contains('Hello')], }, ], -}); +}; + +export default config; ``` -`defineEval()` keeps TypeScript authoring in camelCase and lowers to the canonical snake_case YAML/runtime contract when AgentV loads the `.eval.ts` file. +AgentV loads explicit `*.eval.ts` and `*.eval.mts` files through the same core loader used for YAML evals. The supported TypeScript contract is a default-exported `EvalConfig`. `defineEval(config)` is available as a thin optional helper over the same shape; plain typed default exports are the recommended path. ### Grader helpers Use the `graders` catalog when you want TypeScript helpers for common AgentV grader configs without creating a new eval vocabulary: ```typescript -import { defineEval, graders } from '@agentv/sdk'; +import { graders, type EvalConfig } from '@agentv/sdk'; -export default defineEval({ +const config: EvalConfig = { name: 'grader-helper-suite', prompts: ['{{ input }}'], tests: [ @@ -207,7 +209,9 @@ export default defineEval({ ], }, ], -}); +}; + +export default config; ``` The helpers return ordinary `assert` entries such as `type: contains`, `type: llm-rubric`, and `type: script`. Use the shared `transform` option for assertion-level output shaping. CamelCase SDK options such as `minScore` and `maxSteps` lower to canonical YAML keys such as `min_score` and `max_steps`. @@ -215,7 +219,7 @@ The helpers return ordinary `assert` entries such as `type: contains`, `type: ll If you are coming from Braintrust `scores` or DeepEval metrics, model reusable checks as small AgentV-native helper factories that return these grader configs. They still lower to the same YAML/runtime contract: ```typescript -import { defineEval, graders } from '@agentv/sdk'; +import { graders, type EvalConfig } from '@agentv/sdk'; function ragFaithfulness() { return graders.llmRubric(undefined, { @@ -225,7 +229,7 @@ function ragFaithfulness() { }); } -export default defineEval({ +const config: EvalConfig = { name: 'rag-suite', prompts: ['{{ input }}'], tests: [ @@ -235,7 +239,9 @@ export default defineEval({ assert: [ragFaithfulness()], }, ], -}); +}; + +export default config; ``` Python workflows should emit canonical YAML/JSONL or implement script graders over the stdin/stdout contract. The repo-local helper under `examples/features/sdk-python/` is an example, not a promised published Python package. @@ -248,11 +254,12 @@ Python workflows should emit canonical YAML/JSONL or implement script graders ov - `defineVitestWorkspaceGrader(options)` - Embed the Vitest workspace verifier adapter in a custom script - `defineWorkspaceGrader(handler)` - Define a workspace-aware script grader with file assertion helpers - `definePromptTemplate(handler)` - Define a dynamic prompt template -- `defineEval(definition)` / `evalSuite(definition)` - Define a YAML-aligned `.eval.ts` suite +- `defineEval(config)` - Optional helper for a default-exported TypeScript `EvalConfig` - `graders` - Catalog of built-in AgentV grader config helpers - `containsGrader`, `equalsGrader`, `exactGrader`, `regexGrader`, `isJsonGrader`, `jsonGrader`, `llmRubricGrader`, `scriptGrader` - Named grader helper functions - `toEvalYamlObject(definition)` / `serializeEvalYaml(definition)` - Lower or serialize canonical eval YAML -- `EvalConfig`, `EvalRunResult`, `EvalSummary`, `EvalTestInput`, `EvalAssertionInput` - Programmatic evaluation types +- `EvalConfig` - TypeScript eval config authoring type +- `EvalRunResult`, `EvalSummary`, `EvalTestInput`, `EvalAssertionInput` - Programmatic evaluation types - `AssertionContext`, `AssertionScore` - Assertion types - `ScriptGraderInput`, `ScriptGraderResult`, `Workspace`, `WorkspaceAssertion` - Grader types - `TraceSummary`, `Message`, `ToolCall` - Trace data types diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index b7b4de205..ca5b8eb89 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -211,7 +211,7 @@ export interface EvalRequires { readonly [key: string]: unknown; } -export interface EvalDefinition { +export interface EvalConfig { readonly $schema?: string; readonly name?: string; readonly description?: string; @@ -267,8 +267,8 @@ function lowerEvalYamlValue(value: unknown): unknown { return value; } -function lowerEvalDefinition(definition: unknown): Record { - const lowered = lowerEvalYamlValue(definition) as Record; +function lowerEvalConfig(config: unknown): Record { + const lowered = lowerEvalYamlValue(config) as Record; const { budget_usd: budgetUsd, repeat, ...loweredWithoutRuntimeOptions } = lowered; if (budgetUsd === undefined && repeat === undefined) { return lowered; @@ -293,7 +293,7 @@ function lowerEvalDefinition(definition: unknown): Record { }; } -function attachEvalSuiteBrand(definition: T): T & DefinedEvalSuite { +function attachEvalSuiteBrand(definition: T): T & DefinedEvalSuite { validateTopLevelRuntimeFields(definition); const branded = definition as T & Partial; @@ -319,7 +319,7 @@ function attachEvalSuiteBrand(definition: T): T & Defi return branded as T & DefinedEvalSuite; } -function validateTopLevelRuntimeFields(definition: EvalDefinition): void { +function validateTopLevelRuntimeFields(definition: EvalConfig): void { const rawDefinition = definition as unknown as Record; if (Object.prototype.hasOwnProperty.call(rawDefinition, 'input')) { throw new Error( @@ -362,17 +362,10 @@ function validateTopLevelRuntimeFields(definition: EvalDefinition): void { * non-enumerable lowering hook so AgentV can materialize the canonical * snake_case eval contract when the suite is loaded from a `.eval.ts` file. */ -export function defineEval(definition: T): T & DefinedEvalSuite { +export function defineEval(definition: T): T & DefinedEvalSuite { return attachEvalSuiteBrand(definition); } -/** - * Alias for `defineEval()` when a suite reads more clearly as a plain object. - */ -export function evalSuite(definition: T): T & DefinedEvalSuite { - return defineEval(definition); -} - /** * Lower a TypeScript-authored eval suite into the canonical snake_case object * contract used by YAML files and the runtime loader. @@ -380,17 +373,15 @@ export function evalSuite(definition: T): T & DefinedE * Only known AgentV wire keys are converted. Unknown keys are preserved as-is * so opaque assertion, provider, and metadata payloads are not corrupted. */ -export function toEvalYamlObject( +export function toEvalYamlObject( definition: T, ): LowerEvalYamlValue { - return lowerEvalDefinition(definition) as LowerEvalYamlValue; + return lowerEvalConfig(definition) as LowerEvalYamlValue; } /** * Serialize an eval suite to canonical YAML. */ -export function serializeEvalYaml( - definition: T, -): string { +export function serializeEvalYaml(definition: T): string { return stringifyYaml(toEvalYamlObject(definition), { lineWidth: 0 }).trimEnd(); } diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index f54146432..09685155e 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -131,7 +131,6 @@ export { type AssertEntry, type ConversationTurnInput, type EvalAssertionInput, - type EvalConfig, type EvalRunArtifacts, type EvalRunResult, type EvalSummary, @@ -140,12 +139,11 @@ export { export { defineEval, - evalSuite, serializeEvalYaml, toEvalYamlObject, type DefinedEvalSuite, type EvalAssertionConfig, - type EvalDefinition, + type EvalConfig, type EvalDockerEnvironment, type EvalDockerEnvironmentMount, type EvalDockerEnvironmentResources, diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index d7ce00c4f..60d1fb70b 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'bun:test'; -import { defineEval, evalSuite, serializeEvalYaml, toEvalYamlObject } from '../src/eval.js'; +import { defineEval, serializeEvalYaml, toEvalYamlObject } from '../src/eval.js'; describe('YAML-aligned eval authoring helpers', () => { it('lowers known AgentV fields to canonical snake_case without broad key rewriting', () => { @@ -173,7 +173,7 @@ describe('YAML-aligned eval authoring helpers', () => { }); it('serializes canonical YAML and uses the assert block', () => { - const suite = evalSuite({ + const suite = defineEval({ name: 'yaml-round-trip', prompts: ['{{ input }}'], tests: [ diff --git a/packages/sdk/test/package-graph.test.ts b/packages/sdk/test/package-graph.test.ts index 55177694b..e43f96f24 100644 --- a/packages/sdk/test/package-graph.test.ts +++ b/packages/sdk/test/package-graph.test.ts @@ -48,4 +48,16 @@ describe('core/sdk package graph', () => { expect(offenders).toEqual([]); }); + + it('exports EvalConfig as the single TypeScript eval authoring type', () => { + const indexSource = readFileSync(path.join(repoRoot, 'packages/sdk/src/index.ts'), 'utf8'); + const evalSource = readFileSync(path.join(repoRoot, 'packages/sdk/src/eval.ts'), 'utf8'); + + expect(indexSource).toContain('type EvalConfig'); + expect(indexSource).not.toContain('EvalDefinition'); + expect(indexSource).not.toContain('evalSuite'); + expect(evalSource).toContain('export interface EvalConfig'); + expect(evalSource).not.toContain('export interface EvalDefinition'); + expect(evalSource).not.toContain('export function evalSuite'); + }); }); diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 0ea6af87a..ba62c6eb0 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -755,32 +755,36 @@ Use `@agentv/sdk` as the public lightweight SDK package for TypeScript/JavaScrip ### YAML-aligned eval authoring ```typescript -import { defineEval, graders } from '@agentv/sdk'; +// evals/helper-suite.eval.ts +import { graders, type EvalConfig } from '@agentv/sdk'; -export default defineEval({ +const config: EvalConfig = { name: 'helper-suite', target: 'default', - // The SDK helper lowers this to evaluate_options.repeat in generated YAML. + // TypeScript config loading lowers this to evaluate_options.repeat. repeat: { count: 3, strategy: 'pass_any', earlyExit: false, }, threshold: 0.8, + prompts: ['{{ task }}'], tests: [ { id: 'json-answer', - input: 'Return a JSON answer with a status field.', + vars: { task: 'Return a JSON answer with a status field.' }, assert: [ graders.json({ name: 'valid-json', required: true }), graders.regex(/"status"\s*:/, { name: 'status-key' }), ], }, ], -}); +}; + +export default config; ``` -The `graders` catalog returns ordinary `assert` entries such as `type: is-json`, `type: regex`, `type: llm-rubric`, and `type: script`. `defineEval()` lowers camelCase TypeScript fields such as `expectedOutput`, `inputFiles`, and `maxSteps` to canonical snake_case YAML/runtime keys. +The `graders` catalog returns ordinary `assert` entries such as `type: is-json`, `type: regex`, `type: llm-rubric`, and `type: script`. Explicit `*.eval.ts` and `*.eval.mts` files should default-export an `EvalConfig`; `defineEval(config)` is only an optional thin helper over that same shape. TypeScript config loading lowers camelCase fields such as `expectedOutput`, `inputFiles`, and `maxSteps` to canonical snake_case YAML/runtime keys. If adapting Braintrust `scores` or DeepEval metrics, write small AgentV helper factories that return `graders.*` configs: diff --git a/skills-data/agentv-eval-writer/references/custom-evaluators.md b/skills-data/agentv-eval-writer/references/custom-evaluators.md index a01a02707..997f90c91 100644 --- a/skills-data/agentv-eval-writer/references/custom-evaluators.md +++ b/skills-data/agentv-eval-writer/references/custom-evaluators.md @@ -59,13 +59,15 @@ import { createTargetClient, defineScriptGrader, defineEval, + type EvalConfig, definePromptTemplate, graders, } from '@agentv/sdk'; ``` - `defineScriptGrader(fn)` - Wraps evaluation function with stdin/stdout handling -- `defineEval(definition)` - Defines a YAML-aligned `.eval.ts` suite +- `EvalConfig` - Public TypeScript eval authoring type for default-exported `*.eval.ts` and `*.eval.mts` files +- `defineEval(definition)` - Optional thin helper over the same `EvalConfig` shape - `graders` - Helper catalog that returns ordinary AgentV `assert` entries - `createTargetClient()` - Returns LLM proxy client (when `target: {}` configured) - `.invoke({question, systemPrompt})` - Single LLM call @@ -81,7 +83,7 @@ For Python, the repo-local helper example in `examples/features/sdk-python/` kee Use helper factories for reusable Braintrust/DeepEval-inspired checks, but keep the result as AgentV `assert` entries: ```typescript -import { defineEval, graders } from '@agentv/sdk'; +import { graders, type EvalConfig } from '@agentv/sdk'; function ragFaithfulness() { return graders.llmRubric(undefined, { @@ -91,19 +93,22 @@ function ragFaithfulness() { }); } -export default defineEval({ +const config: EvalConfig = { name: 'rag-suite', + prompts: ['{{ task }}'], tests: [ { id: 'grounded-answer', - input: 'Answer using the retrieved context.', + vars: { task: 'Answer using the retrieved context.' }, assert: [ graders.contains('source', { name: 'mentions-source' }), ragFaithfulness(), ], }, ], -}); +}; + +export default config; ``` The helper lowers to ordinary YAML: