From c2d5e12eccdb541e1c48f1ac20b1036cef54a4a0 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 08:06:41 +0200 Subject: [PATCH] feat(config): lower scenarios into tests --- .../evaluation/validation/eval-file.schema.ts | 28 +- .../evaluation/validation/eval-validator.ts | 236 +++++++++ packages/core/src/evaluation/yaml-parser.ts | 170 +++++++ .../evaluation/loaders/jsonl-parser.test.ts | 224 +++++++++ .../validation/eval-file-schema.test.ts | 58 +++ .../validation/eval-validator.test.ts | 203 ++++++++ .../references/eval.schema.json | 447 +++++++++++++++++- 7 files changed, 1350 insertions(+), 16 deletions(-) diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index e493bf6c2..bc56c77b6 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -836,25 +836,13 @@ const ConfigDefaultsSchema = z }) .strict(); -const ScenarioConfigSchema = z - .object({ - vars: JsonObjectSchema.optional(), - provider: EvalTargetSchema.optional(), - providers: EvalTargetsSchema.optional(), - prompts: PromptsSchema.optional(), - provider_output: ExpectedOutputSchema.optional(), - assert: z.array(AssertionItemSchema).optional(), - options: JsonObjectSchema.optional(), - threshold: z.number().min(0).max(1).optional(), - metadata: z.record(z.unknown()).optional(), - }) - .passthrough(); +const ScenarioConfigSchema = EvalTestSchema.partial(); const ScenarioSchema = z .object({ description: z.string().optional(), - config: z.array(ScenarioConfigSchema).optional(), - tests: z.array(EvalTestSchema).optional(), + config: z.array(ScenarioConfigSchema), + tests: z.array(EvalTestSchema), }) .strict(); @@ -900,6 +888,16 @@ export const EvalFileSchemaInput: z.ZodType = z.object({ imports: z.never({ invalid_type_error: TOP_LEVEL_IMPORTS_MESSAGE }).optional(), // Tests (inline raw cases, legacy include entries, or external raw-case path) tests: TestsSchema.optional(), + providerPromptMap: z + .never({ + invalid_type_error: "Top-level 'providerPromptMap' is not supported. Use 'targets'.", + }) + .optional(), + provider_prompt_map: z + .never({ + invalid_type_error: "Top-level 'provider_prompt_map' is not supported. Use 'targets'.", + }) + .optional(), // Shared composable config graph fields graders: z.union([z.array(ConfigGraderSchema), z.string().min(1)]).optional(), defaults: z.union([ConfigDefaultsSchema, z.string().min(1)]).optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 6454c8a60..42fa11b78 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -335,6 +335,8 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ 'preprocessors', "Top-level 'preprocessors' has been removed from authored eval YAML. Use default_test.options.transform or assertion-level transform instead.", ], + ['providerPromptMap', "Top-level 'providerPromptMap' is not supported. Use 'targets'."], + ['provider_prompt_map', "Top-level 'provider_prompt_map' is not supported. Use 'targets'."], ]); /** Deprecated top-level fields with migration hints. */ @@ -374,6 +376,7 @@ const KNOWN_TEST_FIELDS = new Set([ 'window_size', ]); +const KNOWN_SCENARIO_FIELDS = new Set(['description', 'config', 'tests']); const SUPPORTED_WORKSPACE_REPO_FIELDS = new Set(['path', 'repo', 'commit', 'ancestor', 'sparse']); const KNOWN_REMOVED_WORKSPACE_FIELDS = new Set([ 'template', @@ -393,6 +396,22 @@ REMOVED_TEST_FIELDS.set( 'input', "tests[].input has been removed from authored eval YAML. Put prompt text or chat/system/user messages in top-level 'prompts' and put row-specific data in tests[].vars.", ); +REMOVED_TEST_FIELDS.set( + 'preprocessors', + 'tests[].preprocessors has been removed from authored eval YAML. Use tests[].options.transform or assertion-level transform instead.', +); +REMOVED_TEST_FIELDS.set( + 'postprocess', + 'tests[].postprocess has been removed from authored eval YAML. Use tests[].options.transform instead.', +); +REMOVED_TEST_FIELDS.set( + 'eval_cases', + "tests[].eval_cases has been removed from authored eval YAML. Use 'tests' rows directly.", +); +REMOVED_TEST_FIELDS.set( + 'evalcases', + "tests[].evalcases has been removed from authored eval YAML. Use 'tests' rows directly.", +); /** Deprecated test-level fields with migration hints. */ const DEPRECATED_TEST_FIELDS = new Map([ @@ -570,6 +589,7 @@ export async function validateEvalFile(filePath: string): Promise 0; const hasScenarios = Array.isArray(parsed.scenarios); @@ -1437,6 +1457,222 @@ function validateDefaultTest( } } +async function validateScenarios( + scenarios: JsonValue | undefined, + parsed: JsonObject, + filePath: string, + errors: ValidationError[], + customAssertionTypes: ReadonlySet, +): Promise { + if (scenarios === undefined) { + return; + } + if (!Array.isArray(scenarios)) { + errors.push({ + severity: 'error', + filePath, + location: 'scenarios', + message: "Invalid 'scenarios' field (must be an array)", + }); + return; + } + + for (let scenarioIndex = 0; scenarioIndex < scenarios.length; scenarioIndex++) { + const scenario = scenarios[scenarioIndex]; + const scenarioLocation = `scenarios[${scenarioIndex}]`; + if (!isObject(scenario)) { + errors.push({ + severity: 'error', + filePath, + location: scenarioLocation, + message: 'Scenario must be an object', + }); + continue; + } + + for (const key of Object.keys(scenario)) { + if (!KNOWN_SCENARIO_FIELDS.has(key)) { + errors.push({ + severity: 'error', + filePath, + location: `${scenarioLocation}.${key}`, + message: `Unknown scenario field '${key}'.`, + }); + } + } + + if (!Array.isArray(scenario.config)) { + errors.push({ + severity: 'error', + filePath, + location: `${scenarioLocation}.config`, + message: "Invalid 'config' field (must be an array)", + }); + } else { + for (let configIndex = 0; configIndex < scenario.config.length; configIndex++) { + await validateScenarioTestLikeRow( + scenario.config[configIndex], + `${scenarioLocation}.config[${configIndex}]`, + filePath, + errors, + customAssertionTypes, + { requireInput: false }, + ); + } + } + + const scenarioConfigs = Array.isArray(scenario.config) ? scenario.config : []; + + if (!Array.isArray(scenario.tests)) { + errors.push({ + severity: 'error', + filePath, + location: `${scenarioLocation}.tests`, + message: "Invalid 'tests' field (must be an array)", + }); + continue; + } + + for (let testIndex = 0; testIndex < scenario.tests.length; testIndex++) { + const test = scenario.tests[testIndex]; + const everyConfigProvidesInput = + scenarioConfigs.length > 0 && + scenarioConfigs.every( + (config) => + isObject(config) && + (config.prompts !== undefined || config.provider_output !== undefined), + ); + const requireInput = + isObject(test) && + parsed.prompts === undefined && + test.prompts === undefined && + test.provider_output === undefined && + !everyConfigProvidesInput; + await validateScenarioTestLikeRow( + test, + `${scenarioLocation}.tests[${testIndex}]`, + filePath, + errors, + customAssertionTypes, + { requireInput }, + ); + } + } +} + +async function validateScenarioTestLikeRow( + row: JsonValue | undefined, + location: string, + filePath: string, + errors: ValidationError[], + customAssertionTypes: ReadonlySet, + options: { readonly requireInput: boolean }, +): Promise { + if (!isObject(row)) { + errors.push({ + severity: 'error', + filePath, + location, + message: 'Scenario row must be an object', + }); + return; + } + + for (const key of Object.keys(row)) { + const removedMessage = REMOVED_TEST_FIELDS.get(key); + if (removedMessage) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.${key}`, + message: removedMessage.replaceAll('tests[]', location), + }); + continue; + } + const deprecationMessage = DEPRECATED_TEST_FIELDS.get(key); + if (deprecationMessage) { + errors.push({ + severity: 'warning', + filePath, + location: `${location}.${key}`, + message: deprecationMessage, + }); + } else if (!KNOWN_TEST_FIELDS.has(key)) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.${key}`, + message: `Unknown test field '${key}'.`, + }); + } + } + + const id = row.id; + if (id !== undefined && (typeof id !== 'string' || id.trim().length === 0)) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.id`, + message: "Invalid 'id' field (must be a non-empty string when provided)", + }); + } + + const criteria = row.criteria; + if (criteria !== undefined && (typeof criteria !== 'string' || criteria.trim().length === 0)) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.criteria`, + message: "Invalid 'criteria' field (must be a non-empty string if provided)", + }); + } else if (criteria !== undefined && row.assert !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.criteria`, + message: + "Do not combine test-level 'criteria' with 'assert'. Put human-readable case descriptions in tests[].description, or express grading text as an explicit assertion such as { type: 'llm-rubric', value: ... }.", + }); + } + + validateInputField(row.input, `${location}.input`, filePath, errors, { + required: options.requireInput, + }); + + if (row.expected_output !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.expected_output`, + message: + "tests[].expected_output has been removed from authored eval YAML. Put the reference answer in tests[].vars.expected_output and consume it with an explicit assertion, for example { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }.", + }); + } + + validateAssertArray(row.assert, `${location}.assert`, filePath, errors, customAssertionTypes); + validateRunOverride(row.run, `${location}.run`, filePath, errors); + validateTestOptions(row.options, `${location}.options`, filePath, errors); + await validateEnvironmentConfig(row.environment, filePath, errors, `${location}.environment`); + validateConversationMode(row, location, filePath, errors); + await validateWorkspaceConfig(row.workspace, filePath, errors, `${location}.workspace`); + + const caseExecution = isObject(row.execution) ? row.execution : undefined; + if (caseExecution) { + validateTestExecutionFields(caseExecution, filePath, errors, location); + rejectRuntimeWorkspaceConfig( + caseExecution.workspace, + filePath, + errors, + `${location}.execution.workspace`, + ); + } + + const targets = row.providers !== undefined ? row.providers : row.provider; + if (targets !== undefined) { + validateTargetTestbedFields(targets, `${location}.providers`, filePath, errors); + } +} + function validateEvaluateOptions( evaluateOptions: JsonValue | undefined, location: string, diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 085e6eec7..cea22a086 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -192,6 +192,7 @@ function formatCircularImportChain( type RawTestSuite = JsonObject & { readonly tests?: JsonValue; + readonly scenarios?: JsonValue; readonly target?: JsonValue; readonly providers?: JsonValue; readonly model?: JsonValue; @@ -256,6 +257,12 @@ type RawEvalCase = JsonObject & { readonly window_size?: JsonValue; }; +type RawScenario = JsonObject & { + readonly description?: JsonValue; + readonly config?: JsonValue; + readonly tests?: JsonValue; +}; + type PromptDefinition = { readonly identity: EvalPromptIdentity; readonly input: JsonValue; @@ -284,12 +291,160 @@ function rejectRemovedEvalCasesAliases(suite: RawTestSuite, evalFilePath: string } } +function rejectRemovedScenarioRowFields(row: JsonObject, location: string): void { + if (row.eval_cases !== undefined) { + throw new Error(`${location}.eval_cases has been removed. Use ${location} fields directly.`); + } + if (row.evalcases !== undefined) { + throw new Error(`${location}.evalcases has been removed. Use ${location} fields directly.`); + } + if (row.input !== undefined) { + throw new Error( + `${location}.input has been removed from authored eval YAML. Put prompt text or chat/system/user messages in top-level 'prompts' and put row-specific data in ${location}.vars.`, + ); + } + if (row.expected_output !== undefined) { + throw new Error( + `${location}.expected_output has been removed from authored eval YAML. Put the reference answer in ${location}.vars.expected_output and consume it with an explicit assertion such as { type: 'llm-rubric', value: 'Matches the reference answer: {{ expected_output }}' }.`, + ); + } + rejectPostprocess(row, location); + rejectPreprocessors(row, location); + rejectPostprocess(row.options, `${location}.options`); + if (Array.isArray(row.assert)) { + row.assert.forEach((assertion, assertionIndex) => { + rejectPostprocess(assertion, `${location}.assert[${assertionIndex}]`); + rejectPreprocessors(assertion, `${location}.assert[${assertionIndex}]`); + }); + } +} + function resolveTests(suite: RawTestSuite, evalFilePath: string): JsonValue | undefined { rejectRemovedEvalCasesAliases(suite, evalFilePath); if (suite.tests !== undefined) return suite.tests; return undefined; } +function mergeJsonObjectFields( + first: JsonValue | undefined, + second: JsonValue | undefined, +): JsonObject | undefined { + const merged = { + ...(isJsonObject(first) ? first : {}), + ...(isJsonObject(second) ? second : {}), + }; + return Object.keys(merged).length > 0 ? merged : undefined; +} + +function concatJsonArrayFields( + first: JsonValue | undefined, + second: JsonValue | undefined, +): readonly JsonValue[] | undefined { + const merged = [ + ...(Array.isArray(first) ? first : first !== undefined ? [first] : []), + ...(Array.isArray(second) ? second : second !== undefined ? [second] : []), + ]; + return merged.length > 0 ? merged : undefined; +} + +function mergeScenarioTestRows( + config: JsonObject, + test: JsonObject, + scenarioIndex: number, + configIndex: number, + testIndex: number, + testCount: number, +): JsonObject { + const vars = mergeJsonObjectFields(config.vars, test.vars); + const metadata = mergeJsonObjectFields(config.metadata, test.metadata); + const options = mergeJsonObjectFields(config.options, test.options); + const run = mergeJsonObjectFields(config.run, test.run); + const assertions = concatJsonArrayFields(config.assert, test.assert); + + const id = + typeof test.id === 'string' && test.id.trim().length > 0 + ? test.id + : typeof config.id === 'string' && config.id.trim().length > 0 + ? testCount > 1 + ? `${config.id}__scenario_test_${testIndex + 1}` + : config.id + : stableScenarioTestId(scenarioIndex, configIndex, testIndex); + + return { + ...config, + ...test, + ...(vars ? { vars } : {}), + ...(metadata ? { metadata } : {}), + ...(options ? { options } : {}), + ...(run ? { run } : {}), + ...(assertions ? { assert: assertions } : {}), + id, + }; +} + +function lowerScenariosIntoTests(suite: RawTestSuite, evalFilePath: string): readonly JsonValue[] { + const scenarios = suite.scenarios; + if (scenarios === undefined) { + return []; + } + if (!Array.isArray(scenarios)) { + throw new Error(`Invalid eval file ${evalFilePath}: scenarios must be an array.`); + } + + const lowered: JsonValue[] = []; + for (let scenarioIndex = 0; scenarioIndex < scenarios.length; scenarioIndex++) { + const scenario = scenarios[scenarioIndex]; + if (!isJsonObject(scenario)) { + throw new Error( + `Invalid eval file ${evalFilePath}: scenarios[${scenarioIndex}] must be an object.`, + ); + } + + const rawScenario = scenario as RawScenario; + if (!Array.isArray(rawScenario.config)) { + throw new Error( + `Invalid eval file ${evalFilePath}: scenarios[${scenarioIndex}].config must be an array.`, + ); + } + if (!Array.isArray(rawScenario.tests)) { + throw new Error( + `Invalid eval file ${evalFilePath}: scenarios[${scenarioIndex}].tests must be an array.`, + ); + } + + for (let configIndex = 0; configIndex < rawScenario.config.length; configIndex++) { + const config = rawScenario.config[configIndex]; + if (!isJsonObject(config)) { + throw new Error( + `Invalid eval file ${evalFilePath}: scenarios[${scenarioIndex}].config[${configIndex}] must be an object.`, + ); + } + rejectRemovedScenarioRowFields(config, `scenarios[${scenarioIndex}].config[${configIndex}]`); + for (let testIndex = 0; testIndex < rawScenario.tests.length; testIndex++) { + const test = rawScenario.tests[testIndex]; + if (!isJsonObject(test)) { + throw new Error( + `Invalid eval file ${evalFilePath}: scenarios[${scenarioIndex}].tests[${testIndex}] must be an object.`, + ); + } + rejectRemovedScenarioRowFields(test, `scenarios[${scenarioIndex}].tests[${testIndex}]`); + lowered.push( + mergeScenarioTestRows( + config, + test, + scenarioIndex, + configIndex, + testIndex, + rawScenario.tests.length, + ), + ); + } + } + } + + return lowered; +} + function interpolateCaseField( value: T, vars: JsonObject | undefined, @@ -395,6 +550,14 @@ function stablePromptId(value: unknown): string { return createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 12); } +function stableScenarioTestId( + scenarioIndex: number, + configIndex: number, + testIndex: number, +): string { + return `scenario-${scenarioIndex + 1}-${configIndex + 1}-${testIndex + 1}`; +} + function safePromptId(value: string): string { const safe = value .trim() @@ -1325,10 +1488,17 @@ async function loadTestsFromParsedYamlValue( }); expandedTestCases = expanded.rawCases; importedSuiteTests.push(...expanded.importedSuiteTests); + } else if (suite.scenarios !== undefined) { + expandedTestCases = []; } else { throw new Error(`Invalid test file format: ${evalFilePath} - missing 'tests' field`); } + const scenarioTestCases = lowerScenariosIntoTests(suite, evalFilePath); + if (scenarioTestCases.length > 0) { + expandedTestCases = [...expandedTestCases, ...scenarioTestCases]; + } + expandedTestCases = mergeDefaultTestVarsIntoCases(expandedTestCases, suite.default_test); expandedTestCases = mergeDefaultTestOptionsIntoCases(expandedTestCases, suite.default_test); diff --git a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts index 152b696be..d8d34a055 100644 --- a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts +++ b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts @@ -894,6 +894,230 @@ tests: }); }); +describe('Promptfoo scenarios (YAML)', () => { + let tempDir: string; + + beforeAll(async () => { + tempDir = path.join(os.tmpdir(), `agentv-test-scenarios-${Date.now()}`); + await mkdir(tempDir, { recursive: true }); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('loads scenarios-only files as ordinary tests', async () => { + const yamlPath = path.join(tempDir, 'scenarios-only.yaml'); + await writeFile( + yamlPath, + `prompts: + - "Review {{ severity }} {{ diff }}" +scenarios: + - config: + - vars: + severity: high + tests: + - vars: + diff: critical fix + assert: + - type: contains + value: critical +`, + ); + + const cases = await loadTests(yamlPath, tempDir); + + expect(cases).toHaveLength(1); + expect(cases[0].id).toBe('scenario-1-1-1'); + expect(cases[0].question).toBe('Review high critical fix'); + expect(cases[0].vars).toEqual({ severity: 'high', diff: 'critical fix' }); + expect(cases[0].assertions?.[0]).toMatchObject({ type: 'contains', value: 'critical' }); + }); + + it('appends lowered scenarios after normal tests', async () => { + const yamlPath = path.join(tempDir, 'tests-and-scenarios.yaml'); + await writeFile( + yamlPath, + `prompts: + - "Review {{ diff }}" +tests: + - id: canonical + vars: + diff: ordinary change + assert: + - type: contains + value: ordinary +scenarios: + - config: + - vars: + severity: high + tests: + - vars: + diff: critical fix + assert: + - type: contains + value: critical +`, + ); + + const cases = await loadTests(yamlPath, tempDir); + + expect(cases.map((test) => test.id)).toEqual(['canonical', 'scenario-1-1-1']); + }); + + it('merges default_test, scenario config, and scenario test fields in precedence order', async () => { + const yamlPath = path.join(tempDir, 'scenario-merge.yaml'); + await writeFile( + yamlPath, + `prompts: + - "Review {{ shared }} {{ default_only }} {{ config_only }} {{ test_only }}" +default_test: + vars: + shared: default + default_only: base + options: + repeat: + count: 2 + assert: + - type: contains + value: default assertion +scenarios: + - config: + - vars: + shared: config + config_only: config value + metadata: + owner: config + source: scenario-config + options: + repeat: + count: 3 + run: + timeout_seconds: 10 + assert: + - type: contains + value: config assertion + tests: + - vars: + shared: test + test_only: test value + metadata: + owner: test + options: + repeat: + count: 4 + run: + threshold: 0.8 + assert: + - type: contains + value: test assertion +`, + ); + + const cases = await loadTests(yamlPath, tempDir); + + expect(cases).toHaveLength(1); + expect(cases[0].vars).toEqual({ + shared: 'test', + default_only: 'base', + config_only: 'config value', + test_only: 'test value', + }); + expect(cases[0].metadata).toMatchObject({ + owner: 'test', + source: 'scenario-config', + }); + expect(cases[0].run).toMatchObject({ + repeat: { count: 4, strategy: 'pass_any' }, + timeoutSeconds: 10, + threshold: 0.8, + }); + expect(cases[0].assertions?.map((assertion) => assertion.value)).toEqual([ + 'config assertion', + 'test assertion', + 'default assertion', + ]); + }); + + it('expands top-level prompt matrices through lowered scenarios', async () => { + const yamlPath = path.join(tempDir, 'scenario-prompts.yaml'); + await writeFile( + yamlPath, + `prompts: + - id: alpha + raw: "Alpha {{ diff }}" + - id: beta + raw: "Beta {{ diff }}" +scenarios: + - config: + - vars: + severity: high + tests: + - vars: + diff: critical fix + assert: + - type: contains + value: fix +`, + ); + + const cases = await loadTests(yamlPath, tempDir); + + expect(cases.map((test) => test.id)).toEqual([ + 'scenario-1-1-1__prompt_alpha', + 'scenario-1-1-1__prompt_beta', + ]); + expect(cases.map((test) => test.testId)).toEqual(['scenario-1-1-1', 'scenario-1-1-1']); + expect(cases.map((test) => test.question)).toEqual(['Alpha critical fix', 'Beta critical fix']); + }); + + it('rejects malformed scenarios at load time', async () => { + const yamlPath = path.join(tempDir, 'malformed-scenario.yaml'); + await writeFile( + yamlPath, + `prompts: + - "Review {{ diff }}" +scenarios: + - tests: + - vars: + diff: critical fix + assert: + - type: contains + value: critical +`, + ); + + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + 'scenarios[0].config must be an array', + ); + }); + + it('hard-errors removed fields inside scenarios at load time', async () => { + const yamlPath = path.join(tempDir, 'scenario-removed-field.yaml'); + await writeFile( + yamlPath, + `prompts: + - "Review {{ diff }}" +scenarios: + - config: + - vars: + severity: high + tests: + - input: removed + vars: + diff: critical fix + assert: + - type: contains + value: critical +`, + ); + + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + 'scenarios[0].tests[0].input has been removed', + ); + }); +}); + describe('Backward-compat aliases', () => { let tempDir: string; diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 9e6bde31e..279cab109 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -58,6 +58,64 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('accepts scenarios-only promptfoo matrix authoring', () => { + const result = EvalFileSchema.safeParse({ + prompts: ['Classify {{ request }} at {{ severity }} severity'], + scenarios: [ + { + config: [{ vars: { severity: 'high' } }], + tests: [ + { + vars: { request: 'database outage' }, + assert: [{ type: 'contains', value: 'outage' }], + }, + ], + }, + ], + }); + + expect(result.success).toBe(true); + }); + + it('requires scenario config and tests arrays', () => { + const result = EvalFileSchema.safeParse({ + prompts: ['Classify {{ request }}'], + scenarios: [ + { + tests: [{ vars: { request: 'database outage' } }], + }, + { + config: [{ vars: { severity: 'high' } }], + }, + ], + }); + + expect(result.success).toBe(false); + }); + + it('rejects removed fields inside scenario config and tests', () => { + const result = EvalFileSchema.safeParse({ + prompts: ['Classify {{ request }}'], + scenarios: [ + { + config: [{ input: 'removed', vars: { severity: 'high' } }], + tests: [{ expected_output: 'removed', vars: { request: 'database outage' } }], + }, + ], + }); + + expect(result.success).toBe(false); + }); + + it('rejects providerPromptMap baggage', () => { + const result = EvalFileSchema.safeParse({ + providerPromptMap: { local: ['prompt-a'] }, + tests: [baseTest], + }); + + expect(result.success).toBe(false); + }); + it('rejects removed eval_cases and evalcases aliases as test collections', () => { const result = EvalFileSchema.safeParse({ eval_cases: [baseTest], diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 942cb0c61..b02493db4 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -501,6 +501,209 @@ extensions: expect(result.errors).toHaveLength(0); }); + it('validates scenarios-only files', async () => { + const filePath = path.join(tempDir, 'scenarios-only.yaml'); + await writeFile( + filePath, + `prompts: + - "Review {{ vars.diff }} with {{ vars.severity }} severity" +scenarios: + - description: severity variants + config: + - vars: + severity: high + tests: + - vars: + diff: critical fix + assert: + - type: contains + value: critical +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('validates scenario tests when scenario config supplies prompts', async () => { + const filePath = path.join(tempDir, 'scenario-config-prompts.yaml'); + await writeFile( + filePath, + `scenarios: + - description: prompt variants + config: + - prompts: + - "Review {{ vars.diff }}" + tests: + - vars: + diff: critical fix + assert: + - type: contains + value: critical +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('rejects scenario tests without input when only some config rows supply prompts', async () => { + const filePath = path.join(tempDir, 'scenario-mixed-config-prompts.yaml'); + await writeFile( + filePath, + `scenarios: + - description: prompt variants + config: + - prompts: + - "Review {{ vars.diff }}" + - vars: + severity: high + tests: + - vars: + diff: critical fix + assert: + - type: contains + value: critical +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'scenarios[0].tests[0].input', + }), + ); + }); + + it('rejects malformed scenarios', async () => { + const filePath = path.join(tempDir, 'malformed-scenarios.yaml'); + await writeFile( + filePath, + `prompts: + - "Review {{ vars.diff }}" +scenarios: + - description: missing config + tests: + - vars: + diff: critical fix + assert: + - type: contains + value: critical + - description: missing tests + config: + - vars: + severity: high +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'scenarios[0].config', + message: expect.stringContaining("Invalid 'config' field"), + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'scenarios[1].tests', + message: expect.stringContaining("Invalid 'tests' field"), + }), + ); + }); + + it('rejects removed fields inside scenarios', async () => { + const filePath = path.join(tempDir, 'scenario-removed-fields.yaml'); + await writeFile( + filePath, + `prompts: + - "Review {{ vars.diff }}" +scenarios: + - config: + - input: removed + options: + postprocess: output.trim() + tests: + - expected_output: removed + preprocessors: + - type: xlsx + vars: + diff: critical fix + assert: + - type: contains + value: critical + postprocess: output.trim() +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ severity: 'error', location: 'scenarios[0].config[0].input' }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'scenarios[0].config[0].options.postprocess', + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'scenarios[0].tests[0].preprocessors', + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'scenarios[0].tests[0].expected_output', + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'scenarios[0].tests[0].assert[0].postprocess', + }), + ); + }); + + it('rejects providerPromptMap baggage', async () => { + const filePath = path.join(tempDir, 'provider-prompt-map.yaml'); + await writeFile( + filePath, + `providerPromptMap: + local: [prompt-a] +tests: + - id: test-1 + criteria: Goal + vars: + input: Query +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'providerPromptMap', + message: expect.stringContaining('providerPromptMap'), + }), + ); + }); + it('rejects top-level providers as a live alias for targets', async () => { const filePath = path.join(tempDir, 'top-level-providers.yaml'); await writeFile( diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index beebc72fc..305863939 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -2067,6 +2067,12 @@ } ] }, + "providerPromptMap": { + "not": {} + }, + "provider_prompt_map": { + "not": {} + }, "graders": { "anyOf": [ { @@ -4765,6 +4771,13 @@ "items": { "type": "object", "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, "vars": { "type": "object", "properties": {}, @@ -5994,6 +6007,18 @@ } ] }, + "input": { + "not": {} + }, + "input_files": { + "type": "array", + "items": { + "type": "string" + } + }, + "expected_output": { + "not": {} + }, "assert": { "type": "array", "items": { @@ -6009,6 +6034,19 @@ ] } }, + "assert_scoring_function": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + }, "options": { "type": "object", "properties": {}, @@ -6019,12 +6057,418 @@ "minimum": 0, "maximum": 1 }, + "execution": { + "type": "object", + "properties": { + "workers": { + "not": {} + }, + "assert": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + } + }, + "skip_defaults": { + "type": "boolean" + }, + "cache": { + "type": "boolean" + }, + "trials": { + "not": {} + }, + "budget_usd": { + "type": "number", + "minimum": 0 + }, + "budgetUsd": { + "type": "number", + "minimum": 0 + }, + "fail_on_error": { + "type": "boolean" + }, + "failOnError": { + "type": "boolean" + }, + "threshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "additionalProperties": false + }, + "run": { + "type": "object", + "properties": { + "threshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "repeat": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "minimum": 1 + }, + "strategy": { + "type": "string", + "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] + }, + "early_exit": { + "type": "boolean" + }, + "cost_limit_usd": { + "type": "number", + "minimum": 0 + } + }, + "required": ["count"], + "additionalProperties": false + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "budget_usd": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "additionalProperties": false + }, + "environment": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*file:\\/\\/" + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, "metadata": { "type": "object", "additionalProperties": {} + }, + "conversation_id": { + "type": "string" + }, + "suite": { + "type": "string" + }, + "depends_on": { + "type": "array", + "items": { + "type": "string" + } + }, + "on_dependency_failure": { + "type": "string", + "enum": ["skip", "fail", "run"] + }, + "mode": { + "type": "string", + "enum": ["conversation"] + }, + "turns": { + "type": "array", + "items": { + "type": "object", + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["text", "file", "image"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + } + ] + } + ] + }, + "expected_output": { + "anyOf": [ + { + "type": "string" + }, + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["text", "file", "image"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + } + ] + } + ] + }, + "assert": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + } + } + }, + "required": ["input"], + "additionalProperties": false + }, + "minItems": 1 + }, + "aggregation": { + "type": "string", + "enum": ["mean", "min", "max"] + }, + "on_turn_failure": { + "type": "string", + "enum": ["continue", "stop"] + }, + "window_size": { + "type": "integer", + "minimum": 1 } }, - "additionalProperties": true + "additionalProperties": false } }, "tests": { @@ -7733,6 +8177,7 @@ } } }, + "required": ["config", "tests"], "additionalProperties": false } },