feat(eval): load explicit TypeScript eval configs

Author: christso

README.md

@@ -240,12 +240,12 @@ const { results, summary } = await evaluate({
 console.log(`${summary.passed}/${summary.total} passed`);
 ```
 
-Use `defineEval()` when you want AgentV to run the TypeScript eval file:
+Use `*.eval.ts` when you want AgentV to run a TypeScript eval config:
 
 ```typescript
-import { defineEval } from '@agentv/sdk';
+import type { EvalConfig } from '@agentv/sdk';
 
-export default defineEval({
+const config: EvalConfig = {
   description: 'Code generation quality',
   tags: { experiment: 'with-skills' },
   target: {
@@ -285,7 +285,9 @@ export default defineEval({
       ],
     },
   ],
-});
+};
+
+export default config;
 ```
 
 ## Documentation

apps/cli/src/commands/eval/shared.ts

@@ -1,6 +1,7 @@
 import { constants } from 'node:fs';
 import { access, stat } from 'node:fs/promises';
 import path from 'node:path';
+import { isTypeScriptEvalConfigFileName, typeScriptEvalConfigGlob } from '@agentv/core';
 import fg from 'fast-glob';
 
 import { isAgentSkillsEvalsJsonFile } from '../read-adapters/agent-skills-evals.js';
@@ -10,7 +11,7 @@ export interface ResolveEvalPathOptions {
 }
 
 function isNativeEvalFile(filePath: string): boolean {
-  return /\.(ya?ml|jsonl|[cm]?ts)$/i.test(filePath);
+  return /\.(ya?ml|jsonl)$/i.test(filePath) || isTypeScriptEvalConfigFileName(filePath);
 }
 
 function shouldInspectJsonPath(filePath: string): boolean {
@@ -79,8 +80,8 @@ export async function resolveEvalPaths(
       if (candidateStats.isDirectory()) {
         // Auto-expand directory to recursive eval file glob
         const filePattern = options.allowReadAdapters
-          ? '{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts,evals.json,*.evals.json}'
-          : '{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts}';
+          ? `{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,${typeScriptEvalConfigGlob()},evals.json,*.evals.json}`
+          : `{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,${typeScriptEvalConfigGlob()}}`;
         const dirGlob = path.posix.join(candidatePath.replace(/\\/g, '/'), `**/${filePattern}`);
         const dirMatches = await fg(dirGlob, {
           absolute: true,

apps/cli/src/commands/validate/index.ts

@@ -40,7 +40,7 @@ async function runValidateCommand(
 
 export const validateCommand = command({
   name: 'validate',
-  description: 'Validate AgentV eval and targets YAML files',
+  description: 'Validate AgentV eval, TypeScript config, and targets files',
   args: {
     paths: restPositionals({
       type: string,

apps/cli/src/commands/validate/validate-files.ts

@@ -5,11 +5,13 @@ import {
   type ValidationResult,
   type ValidationSummary,
   detectFileType,
+  isTypeScriptEvalConfigFileName,
   validateCasesFile,
   validateConfigFile,
   validateEvalFile,
   validateFileReferences,
   validateTargetsFile,
+  validateTypeScriptEvalConfigFile,
   validateWorkspacePaths,
 } from '@agentv/core/evaluation/validation';
 import fg from 'fast-glob';
@@ -35,6 +37,10 @@ export async function validateFiles(paths: readonly string[]): Promise<Validatio
 async function validateSingleFile(filePath: string): Promise<ValidationResult> {
   const absolutePath = path.resolve(filePath);
 
+  if (isTypeScriptEvalConfigFileName(absolutePath)) {
+    return validateTypeScriptEvalConfigFile(absolutePath);
+  }
+
   // Detect file type (now infers from path if $schema is missing)
   const fileType = await detectFileType(absolutePath);
 
@@ -97,12 +103,14 @@ async function expandPaths(paths: readonly string[]): Promise<readonly string[]>
       const stats = await stat(absolutePath);
 
       if (stats.isFile()) {
-        if (isYamlFile(absolutePath)) expanded.add(absolutePath);
+        if (isYamlFile(absolutePath) || isTypeScriptEvalConfigFileName(absolutePath)) {
+          expanded.add(absolutePath);
+        }
         continue;
       }
       if (stats.isDirectory()) {
-        const yamlFiles = await findYamlFiles(absolutePath);
-        for (const f of yamlFiles) expanded.add(f);
+        const evalFiles = await findEvalFiles(absolutePath);
+        for (const f of evalFiles) expanded.add(f);
         continue;
       }
     } catch {
@@ -120,19 +128,21 @@ async function expandPaths(paths: readonly string[]): Promise<readonly string[]>
       followSymbolicLinks: true,
     });
 
-    const yamlMatches = matches.filter((f) => isYamlFile(f));
-    if (yamlMatches.length === 0) {
-      console.warn(`Warning: No YAML files matched pattern: ${inputPath}`);
+    const evalMatches = matches.filter((f) => isYamlFile(f) || isTypeScriptEvalConfigFileName(f));
+    if (evalMatches.length === 0) {
+      console.warn(
+        `Warning: No eval YAML or TypeScript config files matched pattern: ${inputPath}`,
+      );
     }
-    for (const f of yamlMatches) expanded.add(path.normalize(f));
+    for (const f of evalMatches) expanded.add(path.normalize(f));
   }
 
   const sorted = Array.from(expanded);
   sorted.sort();
   return sorted;
 }
 
-async function findYamlFiles(dirPath: string): Promise<readonly string[]> {
+async function findEvalFiles(dirPath: string): Promise<readonly string[]> {
   const results: string[] = [];
 
   try {
@@ -146,9 +156,12 @@ async function findYamlFiles(dirPath: string): Promise<readonly string[]> {
         if (entry.name === 'node_modules' || entry.name.startsWith('.')) {
           continue;
         }
-        const subFiles = await findYamlFiles(fullPath);
+        const subFiles = await findEvalFiles(fullPath);
         results.push(...subFiles);
-      } else if (entry.isFile() && isEvalYamlFile(entry.name)) {
+      } else if (
+        entry.isFile() &&
+        (isEvalYamlFile(entry.name) || isTypeScriptEvalConfigFileName(entry.name))
+      ) {
         results.push(fullPath);
       }
     }

apps/cli/test/commands/eval/discover.test.ts

@@ -0,0 +1,36 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import { discoverEvalFiles } from '../../../src/commands/eval/discover.js';
+
+describe('discoverEvalFiles', () => {
+  let tempDir: string;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-discover-eval-files-'));
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  it('discovers explicit TypeScript eval config files without arbitrary TypeScript configs', async () => {
+    const evalDir = path.join(tempDir, 'evals');
+    mkdirSync(evalDir, { recursive: true });
+
+    const tsFile = path.join(evalDir, 'greeting.eval.ts');
+    const mtsFile = path.join(evalDir, 'module.eval.mts');
+    writeFileSync(tsFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n');
+    writeFileSync(mtsFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n');
+    writeFileSync(path.join(evalDir, 'helper.ts'), 'export const helper = true;\n');
+    writeFileSync(path.join(evalDir, 'agentvconfig.ts'), 'export default { tests: [] };\n');
+    writeFileSync(path.join(evalDir, 'promptfooconfig.ts'), 'export default { tests: [] };\n');
+
+    const discovered = await discoverEvalFiles(tempDir);
+    const relativePaths = discovered.map((file) => file.relativePath);
+
+    expect(relativePaths).toEqual(['evals/greeting.eval.ts', 'evals/module.eval.mts']);
+  });
+});

apps/cli/test/commands/eval/shared.test.ts

@@ -108,6 +108,32 @@ describe('resolveEvalPaths', () => {
     expect(resolved).toEqual([path.normalize(tsFile)]);
   });
 
+  it('does not discover agentvconfig.ts files from directory auto-expansion', async () => {
+    const evalDir = path.join(tempDir, 'evals');
+    mkdirSync(evalDir, { recursive: true });
+
+    const tsFile = path.join(evalDir, 'agentvconfig.ts');
+    const helperFile = path.join(evalDir, 'helper.ts');
+    writeFileSync(tsFile, 'export default { tests: [] }');
+    writeFileSync(helperFile, 'export const helper = true');
+
+    await expect(resolveEvalPaths([tempDir], tempDir)).rejects.toThrow(
+      'No eval files matched any provided paths or globs',
+    );
+  });
+
+  it('does not discover promptfooconfig.ts files from directory auto-expansion', async () => {
+    const evalDir = path.join(tempDir, 'evals');
+    mkdirSync(evalDir, { recursive: true });
+
+    const tsFile = path.join(evalDir, 'promptfooconfig.ts');
+    writeFileSync(tsFile, 'export default { tests: [] }');
+
+    await expect(resolveEvalPaths([tempDir], tempDir)).rejects.toThrow(
+      'No eval files matched any provided paths or globs',
+    );
+  });
+
   it('discovers Agent Skills evals.json from directory auto-expansion when read adapters are enabled', async () => {
     const evalDir = path.join(tempDir, 'skills', 'demo', 'evals');
     mkdirSync(evalDir, { recursive: true });
@@ -144,6 +170,29 @@ describe('resolveEvalPaths', () => {
     expect(resolved).toEqual([path.normalize(tsFile)]);
   });
 
+  it('does not accept arbitrary direct .ts file paths as eval configs', async () => {
+    const tsFile = path.join(tempDir, 'helper.ts');
+    writeFileSync(tsFile, 'export const helper = true');
+
+    await expect(resolveEvalPaths([tsFile], tempDir)).rejects.toThrow(
+      'No eval files matched any provided paths or globs',
+    );
+  });
+
+  it('filters arbitrary TypeScript files from broad globs', async () => {
+    const evalDir = path.join(tempDir, 'evals');
+    mkdirSync(evalDir, { recursive: true });
+
+    const tsFile = path.join(evalDir, 'custom.eval.mts');
+    const helperFile = path.join(evalDir, 'helper.ts');
+    writeFileSync(tsFile, 'export default { tests: [] }');
+    writeFileSync(helperFile, 'export const helper = true');
+
+    const resolved = await resolveEvalPaths(['evals/**/*.ts', 'evals/**/*.mts'], tempDir);
+
+    expect(resolved).toEqual([path.normalize(tsFile)]);
+  });
+
   it('discovers both .yaml and .ts files from directory', async () => {
     const evalDir = path.join(tempDir, 'evals');
     mkdirSync(evalDir, { recursive: true });

apps/cli/test/commands/validate/validate-files.test.ts

@@ -0,0 +1,114 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import { validateFiles } from '../../../src/commands/validate/validate-files.js';
+
+describe('validateFiles TypeScript eval configs', () => {
+  let tempDir: string;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-validate-ts-config-'));
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  it('validates *.eval.ts through the core loader', async () => {
+    const configFile = path.join(tempDir, 'suite.eval.ts');
+    writeFileSync(
+      configFile,
+      `export default {
+  prompts: ['{{ input }}'],
+  tests: [
+    {
+      id: 'hello',
+      vars: { input: 'Say hello' },
+      assert: [{ type: 'contains', value: 'hello' }],
+    },
+  ],
+};
+`,
+    );
+
+    const summary = await validateFiles([configFile]);
+
+    expect(summary.totalFiles).toBe(1);
+    expect(summary.validFiles).toBe(1);
+    expect(summary.results[0].filePath).toBe(path.normalize(configFile));
+  });
+
+  it('reports missing default export errors for TypeScript configs', async () => {
+    const configFile = path.join(tempDir, 'suite.eval.ts');
+    writeFileSync(configFile, 'export const config = { tests: [] };\n');
+
+    const summary = await validateFiles([configFile]);
+
+    expect(summary.invalidFiles).toBe(1);
+    expect(summary.results[0].errors[0].message).toContain('default export');
+  });
+
+  it('reports invalid TypeScript eval contract errors', async () => {
+    const configFile = path.join(tempDir, 'suite.eval.mts');
+    writeFileSync(
+      configFile,
+      `export default {
+  prompts: ['{{ input }}'],
+  providers: ['openai:gpt-5'],
+  tests: [{ id: 'hello', vars: { input: 'Say hello' } }],
+};
+`,
+    );
+
+    const summary = await validateFiles([configFile]);
+
+    expect(summary.invalidFiles).toBe(1);
+    expect(summary.results[0].errors[0].message).toContain("top-level 'providers'");
+  });
+
+  it('expands directories without treating custom assertion files as eval configs', async () => {
+    const assertionsDir = path.join(tempDir, '.agentv', 'assertions');
+    mkdirSync(assertionsDir, { recursive: true });
+    writeFileSync(path.join(assertionsDir, 'custom-check.ts'), 'export default () => true;\n');
+    const configFile = path.join(tempDir, 'suite.eval.ts');
+    writeFileSync(
+      configFile,
+      `export default {
+  prompts: ['{{ input }}'],
+  tests: [
+    {
+      id: 'hello',
+      vars: { input: 'Say hello' },
+      assert: [{ type: 'contains', value: 'hello' }],
+    },
+  ],
+};
+`,
+    );
+
+    const summary = await validateFiles([tempDir]);
+
+    expect(summary.totalFiles).toBe(1);
+    expect(summary.results[0].filePath).toBe(path.normalize(configFile));
+  });
+
+  it('does not validate promptfooconfig.ts as an eval config', async () => {
+    const configFile = path.join(tempDir, 'promptfooconfig.ts');
+    writeFileSync(configFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n');
+
+    const summary = await validateFiles([tempDir]);
+
+    expect(summary.totalFiles).toBe(0);
+  });
+
+  it('does not validate agentvconfig.ts as an eval config', async () => {
+    const configFile = path.join(tempDir, 'agentvconfig.ts');
+    writeFileSync(configFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n');
+
+    const summary = await validateFiles([tempDir]);
+
+    expect(summary.totalFiles).toBe(0);
+  });
+});