From bcafc061724148310e658d6cdb992d0035e75ed0 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 07:45:11 +0200 Subject: [PATCH] fix(eval): reject removed eval_cases aliases --- .../evaluation/validation/eval-file.schema.ts | 18 +- .../evaluation/validation/eval-validator.ts | 10 +- .../validation/file-reference-validator.ts | 8 +- packages/core/src/evaluation/yaml-parser.ts | 37 +- .../evaluation/loaders/jsonl-parser.test.ts | 30 +- .../validation/eval-file-schema.test.ts | 17 + .../validation/eval-validator.test.ts | 41 + .../references/breaking-changes.md | 7 +- .../references/eval.schema.json | 1862 +---------------- 9 files changed, 123 insertions(+), 1907 deletions(-) diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 3cf064322..e493bf6c2 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -903,8 +903,19 @@ export const EvalFileSchemaInput: z.ZodType = z.object({ // Shared composable config graph fields graders: z.union([z.array(ConfigGraderSchema), z.string().min(1)]).optional(), defaults: z.union([ConfigDefaultsSchema, z.string().min(1)]).optional(), - // Deprecated aliases - eval_cases: TestsSchema.optional(), + // Removed legacy aliases + eval_cases: z + .never({ + invalid_type_error: + "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead.", + }) + .optional(), + evalcases: z + .never({ + invalid_type_error: + "Top-level 'evalcases' has been removed from authored eval YAML. Use 'tests' instead.", + }) + .optional(), // Target target: z.union([z.string().min(1), EvalLocalTargetSchema]).optional(), targets: EvalTargetsSchema.optional(), @@ -936,7 +947,6 @@ export const EvalFileSchemaInput: z.ZodType = z.object({ }); export const EvalFileSchema: z.ZodType = EvalFileSchemaInput.refine( - (value) => - value.tests !== undefined || value.eval_cases !== undefined || value.scenarios !== undefined, + (value) => value.tests !== undefined || value.scenarios !== undefined, { message: "Eval files must define 'tests' or 'scenarios'." }, ); diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 02c82b563..6454c8a60 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -312,6 +312,14 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ 'providers', "Top-level 'providers' is not a runtime alias in AgentV eval YAML. Use 'targets' for systems under test; provider names backend kind inside each target.", ], + [ + 'eval_cases', + "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead.", + ], + [ + 'evalcases', + "Top-level 'evalcases' has been removed from authored eval YAML. Use 'tests' instead.", + ], ['repeat', "Top-level 'repeat' has been removed. Use evaluate_options.repeat instead."], ['runs', "Top-level 'runs' has been removed. Use evaluate_options.repeat.count instead."], [ @@ -331,8 +339,6 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ /** Deprecated top-level fields with migration hints. */ const DEPRECATED_TOP_LEVEL_FIELDS = new Map([ - ['eval_cases', "'eval_cases' is deprecated. Use 'tests' instead."], - ['evalcases', "'evalcases' is deprecated. Use 'tests' instead."], ['evaluator', "'evaluator' is deprecated. Use 'assert' instead."], ]); diff --git a/packages/core/src/evaluation/validation/file-reference-validator.ts b/packages/core/src/evaluation/validation/file-reference-validator.ts index 805beb09c..8eccdf15b 100644 --- a/packages/core/src/evaluation/validation/file-reference-validator.ts +++ b/packages/core/src/evaluation/validation/file-reference-validator.ts @@ -50,13 +50,7 @@ export async function validateFileReferences( return errors; } - let cases: JsonValue | undefined = parsed.tests; - if (cases === undefined && 'eval_cases' in parsed) { - cases = parsed.eval_cases; - } - if (cases === undefined && 'evalcases' in parsed) { - cases = parsed.evalcases; - } + const cases: JsonValue | undefined = parsed.tests; if (!Array.isArray(cases)) { return errors; } diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index f70693713..085e6eec7 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -192,10 +192,6 @@ function formatCircularImportChain( type RawTestSuite = JsonObject & { readonly tests?: JsonValue; - /** @deprecated Use `tests` instead */ - readonly eval_cases?: JsonValue; - /** @deprecated Use `tests` instead */ - readonly evalcases?: JsonValue; readonly target?: JsonValue; readonly providers?: JsonValue; readonly model?: JsonValue; @@ -271,16 +267,26 @@ type PromptExpansionResult = { readonly sourceTestIdById: ReadonlyMap; }; -function resolveTests(suite: RawTestSuite): JsonValue | undefined { - if (suite.tests !== undefined) return suite.tests; - if (suite.eval_cases !== undefined) { - logWarning("'eval_cases' is deprecated. Use 'tests' instead."); - return suite.eval_cases; +function removedEvalCasesAliasMessage(alias: 'eval_cases' | 'evalcases'): string { + return `Top-level '${alias}' has been removed from authored eval YAML. Use 'tests' instead.`; +} + +function rejectRemovedEvalCasesAliases(suite: RawTestSuite, evalFilePath: string): void { + if ('eval_cases' in suite) { + throw new Error( + `Invalid eval file ${evalFilePath}: ${removedEvalCasesAliasMessage('eval_cases')}`, + ); } - if (suite.evalcases !== undefined) { - logWarning("'evalcases' is deprecated. Use 'tests' instead."); - return suite.evalcases; + if ('evalcases' in suite) { + throw new Error( + `Invalid eval file ${evalFilePath}: ${removedEvalCasesAliasMessage('evalcases')}`, + ); } +} + +function resolveTests(suite: RawTestSuite, evalFilePath: string): JsonValue | undefined { + rejectRemovedEvalCasesAliases(suite, evalFilePath); + if (suite.tests !== undefined) return suite.tests; return undefined; } @@ -1283,7 +1289,7 @@ async function loadTestsFromParsedYamlValue( const suiteName = suiteNameFromFile && suiteNameFromFile.length > 0 ? suiteNameFromFile : fallbackSuiteName; - const rawTestCases = resolveTests(suite); + const rawTestCases = resolveTests(suite, evalFilePath); const suiteExperimentConfig = normalizeSuiteExperimentConfig(suite); // Top-level `metadata:` is inherited by cases. Suite identity tags are parsed // separately by parseMetadata() and are not case tags. @@ -2275,7 +2281,7 @@ async function loadRawCasesForInclude(includePath: string): Promise undefined); @@ -2507,8 +2513,7 @@ function buildRawInlineTestSnapshots(rawParsed: unknown): Map { return snapshots; } - const rawTests = - rawParsed.tests ?? rawParsed.eval_cases ?? (rawParsed as Record).evalcases; + const rawTests = rawParsed.tests; if (!Array.isArray(rawTests)) { return snapshots; } diff --git a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts index 4717066c8..152b696be 100644 --- a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts +++ b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts @@ -906,8 +906,8 @@ describe('Backward-compat aliases', () => { await rm(tempDir, { recursive: true, force: true }); }); - describe('eval_cases → tests alias (YAML)', () => { - it('supports eval_cases as deprecated alias for tests', async () => { + describe('removed eval_cases/evalcases aliases (YAML)', () => { + it('rejects eval_cases as a removed top-level alias for tests', async () => { const yamlPath = path.join(tempDir, 'eval-cases-alias.yaml'); await writeFile( yamlPath, @@ -923,14 +923,12 @@ prompts: `, ); - const cases = await loadTests(yamlPath, tempDir); - - expect(cases).toHaveLength(1); - expect(cases[0].id).toBe('test-1'); - expect(cases[0].criteria).toBe('Goal'); + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead.", + ); }); - it('supports evalcases as deprecated alias for tests', async () => { + it('rejects evalcases as a removed top-level alias for tests', async () => { const yamlPath = path.join(tempDir, 'evalcases-alias.yaml'); await writeFile( yamlPath, @@ -943,13 +941,12 @@ prompts: `, ); - const cases = await loadTests(yamlPath, tempDir); - - expect(cases).toHaveLength(1); - expect(cases[0].id).toBe('test-1'); + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + "Top-level 'evalcases' has been removed from authored eval YAML. Use 'tests' instead.", + ); }); - it('tests takes precedence over eval_cases', async () => { + it('rejects eval_cases even when canonical tests is present', async () => { const yamlPath = path.join(tempDir, 'cases-precedence.yaml'); await writeFile( yamlPath, @@ -971,10 +968,9 @@ eval_cases: `, ); - const cases = await loadTests(yamlPath, tempDir); - - expect(cases).toHaveLength(1); - expect(cases[0].id).toBe('canonical'); + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead.", + ); }); }); diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 46f995293..9e6bde31e 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -58,6 +58,23 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('rejects removed eval_cases and evalcases aliases as test collections', () => { + const result = EvalFileSchema.safeParse({ + eval_cases: [baseTest], + evalcases: [baseTest], + }); + + expect(result.success).toBe(false); + if (result.success) throw new Error('Expected removed aliases to be rejected'); + const messages = collectIssueMessages(result.error.issues); + expect( + messages.some((message) => message.includes("Top-level 'eval_cases' has been removed")), + ).toBe(true); + expect( + messages.some((message) => message.includes("Top-level 'evalcases' has been removed")), + ).toBe(true); + }); + it('rejects eval-level execution.max_concurrency', () => { const result = EvalFileSchema.safeParse({ execution: { diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 2add45176..942cb0c61 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -531,6 +531,47 @@ tests: ).toBe(true); }); + it('rejects removed eval_cases and evalcases aliases for tests', async () => { + const filePath = path.join(tempDir, 'removed-eval-cases-aliases.yaml'); + await writeFile( + filePath, + `prompts: + - "{{ prompt }}" +tests: + - vars: + prompt: Canonical + assert: + - type: contains + value: Canonical +eval_cases: + - vars: + prompt: Legacy snake case +evalcases: + - vars: + prompt: Legacy collapsed case +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'eval_cases', + message: expect.stringContaining("Top-level 'eval_cases' has been removed"), + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'evalcases', + message: expect.stringContaining("Top-level 'evalcases' has been removed"), + }), + ); + expect(result.errors.some((error) => error.severity === 'warning')).toBe(false); + }); + it('rejects removed top-level repeat controls with migration guidance', async () => { const filePath = path.join(tempDir, 'removed-repeat-fields.yaml'); await writeFile( diff --git a/skills-data/agentv-eval-migrations/references/breaking-changes.md b/skills-data/agentv-eval-migrations/references/breaking-changes.md index dfcc6b423..fc06e9fb1 100644 --- a/skills-data/agentv-eval-migrations/references/breaking-changes.md +++ b/skills-data/agentv-eval-migrations/references/breaking-changes.md @@ -1082,9 +1082,10 @@ rg -n "include:|tests:" path/to/evals ### Compatibility Notes -`eval_cases` remains a deprecated alias in the current schema, but migrated -YAML should use `tests`. The current convention is that runnable suites use -`*.eval.yaml`; reusable raw case files commonly use `*.cases.yaml` or JSONL. +`eval_cases` and `evalcases` have been removed from authored eval YAML. Migrate +them to `tests` before validating or running the suite. The current convention is +that runnable suites use `*.eval.yaml`; reusable raw case files commonly use +`*.cases.yaml` or JSONL. ## Result Artifact Path Changes Are Not Eval YAML Migrations diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 1bb616e43..beebc72fc 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -2120,1864 +2120,10 @@ ] }, "eval_cases": { - "anyOf": [ - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string" - }, - "vars": { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - "provider": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "config": { - "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "transform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": true - } - ] - }, - "providers": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "config": { - "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "transform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "config": { - "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "transform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "provider_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] - }, - "input": { - "not": {} - }, - "input_files": { - "type": "array", - "items": { - "type": "string" - } - }, - "expected_output": { - "not": {} - }, - "assert": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - } - }, - "assert_scoring_function": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "options": { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - "threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "execution": { - "type": "object", - "properties": { - "workers": { - "not": {} - }, - "assert": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - } - }, - "skip_defaults": { - "type": "boolean" - }, - "cache": { - "type": "boolean" - }, - "trials": { - "not": {} - }, - "budget_usd": { - "type": "number", - "minimum": 0 - }, - "budgetUsd": { - "type": "number", - "minimum": 0 - }, - "fail_on_error": { - "type": "boolean" - }, - "failOnError": { - "type": "boolean" - }, - "threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - } - }, - "additionalProperties": false - }, - "run": { - "type": "object", - "properties": { - "threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "repeat": { - "type": "object", - "properties": { - "count": { - "type": "integer", - "minimum": 1 - }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] - }, - "early_exit": { - "type": "boolean" - }, - "cost_limit_usd": { - "type": "number", - "minimum": 0 - } - }, - "required": ["count"], - "additionalProperties": false - }, - "timeout_seconds": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - }, - "budget_usd": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "additionalProperties": false - }, - "environment": { - "anyOf": [ - { - "type": "string", - "pattern": "^\\s*file:\\/\\/" - }, - { - "type": "object", - "properties": { - "workdir": { - "type": "string", - "minLength": 1 - }, - "setup": { - "allOf": [ - {}, - { - "type": "object", - "properties": { - "command": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - }, - "cwd": { - "type": "string", - "minLength": 1 - }, - "timeout_ms": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "required": ["command"], - "additionalProperties": false - } - ] - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "host" - } - }, - "required": ["workdir", "type"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "workdir": { - "type": "string", - "minLength": 1 - }, - "setup": { - "allOf": [ - {}, - { - "type": "object", - "properties": { - "command": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - }, - "cwd": { - "type": "string", - "minLength": 1 - }, - "timeout_ms": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "required": ["command"], - "additionalProperties": false - } - ] - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "docker" - }, - "context": { - "type": "string", - "minLength": 1 - }, - "dockerfile": { - "type": "string", - "minLength": 1 - }, - "image": { - "type": "string", - "minLength": 1 - }, - "resources": { - "type": "object", - "properties": { - "cpus": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - }, - "memory": { - "type": "string", - "minLength": 1 - }, - "disk": { - "type": "string", - "minLength": 1 - }, - "gpu": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "minLength": 1 - } - ] - } - }, - "additionalProperties": false - }, - "mounts": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source": { - "type": "string", - "minLength": 1 - }, - "target": { - "type": "string", - "minLength": 1 - }, - "access": { - "type": "string", - "enum": ["ro", "rw"] - }, - "read_only": { - "type": "boolean" - } - }, - "required": ["source", "target"], - "additionalProperties": false - } - }, - "secrets": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "required": ["workdir", "type"], - "additionalProperties": false - } - ] - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "conversation_id": { - "type": "string" - }, - "suite": { - "type": "string" - }, - "depends_on": { - "type": "array", - "items": { - "type": "string" - } - }, - "on_dependency_failure": { - "type": "string", - "enum": ["skip", "fail", "run"] - }, - "mode": { - "type": "string", - "enum": ["conversation"] - }, - "turns": { - "type": "array", - "items": { - "type": "object", - "properties": { - "input": { - "anyOf": [ - { - "type": "string" - }, - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - ] - }, - "expected_output": { - "anyOf": [ - { - "type": "string" - }, - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - ] - }, - "assert": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - } - } - }, - "required": ["input"], - "additionalProperties": false - }, - "minItems": 1 - }, - "aggregation": { - "type": "string", - "enum": ["mean", "min", "max"] - }, - "on_turn_failure": { - "type": "string", - "enum": ["continue", "stop"] - }, - "window_size": { - "type": "integer", - "minimum": 1 - } - }, - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "include": { - "type": "string", - "minLength": 1 - }, - "type": { - "type": "string", - "enum": ["suite", "tests"] - }, - "select": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - } - ] - }, - { - "type": "object", - "properties": { - "test_ids": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - } - ] - }, - "tags": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - } - ] - }, - "metadata": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - }, - { - "type": "boolean" - }, - { - "type": "array", - "items": { - "type": ["string", "number", "boolean"] - }, - "minItems": 1 - } - ] - } - } - }, - "additionalProperties": false - } - ] - }, - "run": { - "type": "object", - "properties": { - "threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "repeat": { - "type": "object", - "properties": { - "count": { - "type": "integer", - "minimum": 1 - }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] - }, - "early_exit": { - "type": "boolean" - }, - "cost_limit_usd": { - "type": "number", - "minimum": 0 - } - }, - "required": ["count"], - "additionalProperties": false - }, - "timeout_seconds": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - }, - "budget_usd": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "additionalProperties": false - } - }, - "required": ["include", "type"], - "additionalProperties": false - }, - { - "type": "string", - "minLength": 1 - } - ] - } - }, - { - "type": "string", - "minLength": 1 - } - ] + "not": {} + }, + "evalcases": { + "not": {} }, "target": { "anyOf": [