fix(eval): reject removed eval_cases aliases

Author: christso

packages/core/src/evaluation/validation/eval-file.schema.ts

@@ -903,8 +903,19 @@ export const EvalFileSchemaInput: z.ZodType = z.object({
   // Shared composable config graph fields
   graders: z.union([z.array(ConfigGraderSchema), z.string().min(1)]).optional(),
   defaults: z.union([ConfigDefaultsSchema, z.string().min(1)]).optional(),
-  // Deprecated aliases
-  eval_cases: TestsSchema.optional(),
+  // Removed legacy aliases
+  eval_cases: z
+    .never({
+      invalid_type_error:
+        "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead.",
+    })
+    .optional(),
+  evalcases: z
+    .never({
+      invalid_type_error:
+        "Top-level 'evalcases' has been removed from authored eval YAML. Use 'tests' instead.",
+    })
+    .optional(),
   // Target
   target: z.union([z.string().min(1), EvalLocalTargetSchema]).optional(),
   targets: EvalTargetsSchema.optional(),
@@ -936,7 +947,6 @@ export const EvalFileSchemaInput: z.ZodType = z.object({
 });
 
 export const EvalFileSchema: z.ZodType = EvalFileSchemaInput.refine(
-  (value) =>
-    value.tests !== undefined || value.eval_cases !== undefined || value.scenarios !== undefined,
+  (value) => value.tests !== undefined || value.scenarios !== undefined,
   { message: "Eval files must define 'tests' or 'scenarios'." },
 );

packages/core/src/evaluation/validation/eval-validator.ts

@@ -312,6 +312,14 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map<string, string>([
     'providers',
     "Top-level 'providers' is not a runtime alias in AgentV eval YAML. Use 'targets' for systems under test; provider names backend kind inside each target.",
   ],
+  [
+    'eval_cases',
+    "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead.",
+  ],
+  [
+    'evalcases',
+    "Top-level 'evalcases' has been removed from authored eval YAML. Use 'tests' instead.",
+  ],
   ['repeat', "Top-level 'repeat' has been removed. Use evaluate_options.repeat instead."],
   ['runs', "Top-level 'runs' has been removed. Use evaluate_options.repeat.count instead."],
   [
@@ -331,8 +339,6 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map<string, string>([
 
 /** Deprecated top-level fields with migration hints. */
 const DEPRECATED_TOP_LEVEL_FIELDS = new Map<string, string>([
-  ['eval_cases', "'eval_cases' is deprecated. Use 'tests' instead."],
-  ['evalcases', "'evalcases' is deprecated. Use 'tests' instead."],
   ['evaluator', "'evaluator' is deprecated. Use 'assert' instead."],
 ]);
 

packages/core/src/evaluation/validation/file-reference-validator.ts

@@ -50,13 +50,7 @@ export async function validateFileReferences(
     return errors;
   }
 
-  let cases: JsonValue | undefined = parsed.tests;
-  if (cases === undefined && 'eval_cases' in parsed) {
-    cases = parsed.eval_cases;
-  }
-  if (cases === undefined && 'evalcases' in parsed) {
-    cases = parsed.evalcases;
-  }
+  const cases: JsonValue | undefined = parsed.tests;
   if (!Array.isArray(cases)) {
     return errors;
   }

packages/core/src/evaluation/yaml-parser.ts

@@ -192,10 +192,6 @@ function formatCircularImportChain(
 
 type RawTestSuite = JsonObject & {
   readonly tests?: JsonValue;
-  /** @deprecated Use `tests` instead */
-  readonly eval_cases?: JsonValue;
-  /** @deprecated Use `tests` instead */
-  readonly evalcases?: JsonValue;
   readonly target?: JsonValue;
   readonly providers?: JsonValue;
   readonly model?: JsonValue;
@@ -271,16 +267,26 @@ type PromptExpansionResult = {
   readonly sourceTestIdById: ReadonlyMap<string, string>;
 };
 
-function resolveTests(suite: RawTestSuite): JsonValue | undefined {
-  if (suite.tests !== undefined) return suite.tests;
-  if (suite.eval_cases !== undefined) {
-    logWarning("'eval_cases' is deprecated. Use 'tests' instead.");
-    return suite.eval_cases;
+function removedEvalCasesAliasMessage(alias: 'eval_cases' | 'evalcases'): string {
+  return `Top-level '${alias}' has been removed from authored eval YAML. Use 'tests' instead.`;
+}
+
+function rejectRemovedEvalCasesAliases(suite: RawTestSuite, evalFilePath: string): void {
+  if ('eval_cases' in suite) {
+    throw new Error(
+      `Invalid eval file ${evalFilePath}: ${removedEvalCasesAliasMessage('eval_cases')}`,
+    );
   }
-  if (suite.evalcases !== undefined) {
-    logWarning("'evalcases' is deprecated. Use 'tests' instead.");
-    return suite.evalcases;
+  if ('evalcases' in suite) {
+    throw new Error(
+      `Invalid eval file ${evalFilePath}: ${removedEvalCasesAliasMessage('evalcases')}`,
+    );
   }
+}
+
+function resolveTests(suite: RawTestSuite, evalFilePath: string): JsonValue | undefined {
+  rejectRemovedEvalCasesAliases(suite, evalFilePath);
+  if (suite.tests !== undefined) return suite.tests;
   return undefined;
 }
 
@@ -1283,7 +1289,7 @@ async function loadTestsFromParsedYamlValue(
   const suiteName =
     suiteNameFromFile && suiteNameFromFile.length > 0 ? suiteNameFromFile : fallbackSuiteName;
 
-  const rawTestCases = resolveTests(suite);
+  const rawTestCases = resolveTests(suite, evalFilePath);
   const suiteExperimentConfig = normalizeSuiteExperimentConfig(suite);
   // Top-level `metadata:` is inherited by cases. Suite identity tags are parsed
   // separately by parseMetadata() and are not case tags.
@@ -2275,7 +2281,7 @@ async function loadRawCasesForInclude(includePath: string): Promise<readonly Jso
     if (!isJsonObject(raw)) {
       throw new Error(`Imported eval suite must be a YAML object: ${includePath}`);
     }
-    const tests = resolveTests(raw as RawTestSuite);
+    const tests = resolveTests(raw as RawTestSuite, includePath);
     if (typeof tests === 'string') {
       const externalPath = path.resolve(path.dirname(includePath), tests);
       const pathStat = await stat(externalPath).catch(() => undefined);
@@ -2507,8 +2513,7 @@ function buildRawInlineTestSnapshots(rawParsed: unknown): Map<string, string> {
     return snapshots;
   }
 
-  const rawTests =
-    rawParsed.tests ?? rawParsed.eval_cases ?? (rawParsed as Record<string, unknown>).evalcases;
+  const rawTests = rawParsed.tests;
   if (!Array.isArray(rawTests)) {
     return snapshots;
   }

packages/core/test/evaluation/loaders/jsonl-parser.test.ts

@@ -906,8 +906,8 @@ describe('Backward-compat aliases', () => {
     await rm(tempDir, { recursive: true, force: true });
   });
 
-  describe('eval_cases → tests alias (YAML)', () => {
-    it('supports eval_cases as deprecated alias for tests', async () => {
+  describe('removed eval_cases/evalcases aliases (YAML)', () => {
+    it('rejects eval_cases as a removed top-level alias for tests', async () => {
       const yamlPath = path.join(tempDir, 'eval-cases-alias.yaml');
       await writeFile(
         yamlPath,
@@ -923,14 +923,12 @@ prompts:
 `,
       );
 
-      const cases = await loadTests(yamlPath, tempDir);
-
-      expect(cases).toHaveLength(1);
-      expect(cases[0].id).toBe('test-1');
-      expect(cases[0].criteria).toBe('Goal');
+      await expect(loadTests(yamlPath, tempDir)).rejects.toThrow(
+        "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead.",
+      );
     });
 
-    it('supports evalcases as deprecated alias for tests', async () => {
+    it('rejects evalcases as a removed top-level alias for tests', async () => {
       const yamlPath = path.join(tempDir, 'evalcases-alias.yaml');
       await writeFile(
         yamlPath,
@@ -943,13 +941,12 @@ prompts:
 `,
       );
 
-      const cases = await loadTests(yamlPath, tempDir);
-
-      expect(cases).toHaveLength(1);
-      expect(cases[0].id).toBe('test-1');
+      await expect(loadTests(yamlPath, tempDir)).rejects.toThrow(
+        "Top-level 'evalcases' has been removed from authored eval YAML. Use 'tests' instead.",
+      );
     });
 
-    it('tests takes precedence over eval_cases', async () => {
+    it('rejects eval_cases even when canonical tests is present', async () => {
       const yamlPath = path.join(tempDir, 'cases-precedence.yaml');
       await writeFile(
         yamlPath,
@@ -971,10 +968,9 @@ eval_cases:
 `,
       );
 
-      const cases = await loadTests(yamlPath, tempDir);
-
-      expect(cases).toHaveLength(1);
-      expect(cases[0].id).toBe('canonical');
+      await expect(loadTests(yamlPath, tempDir)).rejects.toThrow(
+        "Top-level 'eval_cases' has been removed from authored eval YAML. Use 'tests' instead.",
+      );
     });
   });
 

packages/core/test/evaluation/validation/eval-file-schema.test.ts

@@ -58,6 +58,23 @@ describe('EvalFileSchema input shorthand', () => {
     expect(result.success).toBe(true);
   });
 
+  it('rejects removed eval_cases and evalcases aliases as test collections', () => {
+    const result = EvalFileSchema.safeParse({
+      eval_cases: [baseTest],
+      evalcases: [baseTest],
+    });
+
+    expect(result.success).toBe(false);
+    if (result.success) throw new Error('Expected removed aliases to be rejected');
+    const messages = collectIssueMessages(result.error.issues);
+    expect(
+      messages.some((message) => message.includes("Top-level 'eval_cases' has been removed")),
+    ).toBe(true);
+    expect(
+      messages.some((message) => message.includes("Top-level 'evalcases' has been removed")),
+    ).toBe(true);
+  });
+
   it('rejects eval-level execution.max_concurrency', () => {
     const result = EvalFileSchema.safeParse({
       execution: {

packages/core/test/evaluation/validation/eval-validator.test.ts

@@ -531,6 +531,47 @@ tests:
     ).toBe(true);
   });
 
+  it('rejects removed eval_cases and evalcases aliases for tests', async () => {
+    const filePath = path.join(tempDir, 'removed-eval-cases-aliases.yaml');
+    await writeFile(
+      filePath,
+      `prompts:
+  - "{{ prompt }}"
+tests:
+  - vars:
+      prompt: Canonical
+    assert:
+      - type: contains
+        value: Canonical
+eval_cases:
+  - vars:
+      prompt: Legacy snake case
+evalcases:
+  - vars:
+      prompt: Legacy collapsed case
+`,
+    );
+
+    const result = await validateEvalFile(filePath);
+
+    expect(result.valid).toBe(false);
+    expect(result.errors).toContainEqual(
+      expect.objectContaining({
+        severity: 'error',
+        location: 'eval_cases',
+        message: expect.stringContaining("Top-level 'eval_cases' has been removed"),
+      }),
+    );
+    expect(result.errors).toContainEqual(
+      expect.objectContaining({
+        severity: 'error',
+        location: 'evalcases',
+        message: expect.stringContaining("Top-level 'evalcases' has been removed"),
+      }),
+    );
+    expect(result.errors.some((error) => error.severity === 'warning')).toBe(false);
+  });
+
   it('rejects removed top-level repeat controls with migration guidance', async () => {
     const filePath = path.join(tempDir, 'removed-repeat-fields.yaml');
     await writeFile(

skills-data/agentv-eval-migrations/references/breaking-changes.md

@@ -1082,9 +1082,10 @@ rg -n "include:|tests:" path/to/evals
 
 ### Compatibility Notes
 
-`eval_cases` remains a deprecated alias in the current schema, but migrated
-YAML should use `tests`. The current convention is that runnable suites use
-`*.eval.yaml`; reusable raw case files commonly use `*.cases.yaml` or JSONL.
+`eval_cases` and `evalcases` have been removed from authored eval YAML. Migrate
+them to `tests` before validating or running the suite. The current convention is
+that runnable suites use `*.eval.yaml`; reusable raw case files commonly use
+`*.cases.yaml` or JSONL.
 
 ## Result Artifact Path Changes Are Not Eval YAML Migrations