agentv/apps/cli/test/commands/validate/validate-files.test.ts at 0c0e375ad58dfd4fae01e85292f58e2d52a72cdb · EntityProcess/agentv · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';

import { validateFiles } from '../../../src/commands/validate/validate-files.js';

describe('validateFiles TypeScript eval configs', () => {
  let tempDir: string;

  beforeEach(() => {
    tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-validate-ts-config-'));
  });

  afterEach(() => {
    rmSync(tempDir, { recursive: true, force: true });
  });

  it('validates *.eval.ts through the core loader', async () => {
    const configFile = path.join(tempDir, 'suite.eval.ts');
    writeFileSync(
      configFile,
      `export default {
  prompts: ['{{ input }}'],
  tests: [
    {
      id: 'hello',
      vars: { input: 'Say hello' },
      assert: [{ type: 'contains', value: 'hello' }],
    },
  ],
};
`,
    );

    const summary = await validateFiles([configFile]);

    expect(summary.totalFiles).toBe(1);
    expect(summary.validFiles).toBe(1);
    expect(summary.results[0].filePath).toBe(path.normalize(configFile));
  });

  it('reports missing default export errors for TypeScript configs', async () => {
    const configFile = path.join(tempDir, 'suite.eval.ts');
    writeFileSync(configFile, 'export const config = { tests: [] };\n');

    const summary = await validateFiles([configFile]);

    expect(summary.invalidFiles).toBe(1);
    expect(summary.results[0].errors[0].message).toContain('default export');
  });

  it('reports invalid TypeScript eval contract errors', async () => {
    const configFile = path.join(tempDir, 'suite.eval.mts');
    writeFileSync(
      configFile,
      `export default {
  prompts: ['{{ input }}'],
  providers: ['openai:gpt-5'],
  tests: [{ id: 'hello', vars: { input: 'Say hello' } }],
};
`,
    );

    const summary = await validateFiles([configFile]);

    expect(summary.invalidFiles).toBe(1);
    expect(summary.results[0].errors[0].message).toContain("top-level 'providers'");
  });

  it('expands directories without treating custom assertion files as eval configs', async () => {
    const assertionsDir = path.join(tempDir, '.agentv', 'assertions');
    mkdirSync(assertionsDir, { recursive: true });
    writeFileSync(path.join(assertionsDir, 'custom-check.ts'), 'export default () => true;\n');
    const configFile = path.join(tempDir, 'suite.eval.ts');
    writeFileSync(
      configFile,
      `export default {
  prompts: ['{{ input }}'],
  tests: [
    {
      id: 'hello',
      vars: { input: 'Say hello' },
      assert: [{ type: 'contains', value: 'hello' }],
    },
  ],
};
`,
    );

    const summary = await validateFiles([tempDir]);

    expect(summary.totalFiles).toBe(1);
    expect(summary.results[0].filePath).toBe(path.normalize(configFile));
  });

  it('does not validate promptfooconfig.ts as an eval config', async () => {
    const configFile = path.join(tempDir, 'promptfooconfig.ts');
    writeFileSync(configFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n');

    const summary = await validateFiles([tempDir]);

    expect(summary.totalFiles).toBe(0);
  });

  it('does not validate agentvconfig.ts as an eval config', async () => {
    const configFile = path.join(tempDir, 'agentvconfig.ts');
    writeFileSync(configFile, 'export default { prompts: ["{{ input }}"], tests: [] };\n');

    const summary = await validateFiles([tempDir]);

    expect(summary.totalFiles).toBe(0);
  });
});