Skip to content

Commit ba7993e

Browse files
christsochristso
authored
fix(eval): reject top-level experiment authoring (#1728)
1 parent 91fb908 commit ba7993e

15 files changed

Lines changed: 113 additions & 71 deletions

File tree

‎apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,10 @@ agentv eval evals/my-eval.yaml --experiment without_skills
6666
```
6767

6868
The experiment label chooses the result bucket and is propagated to each entry
69-
in `index.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval
70-
file. If neither is set, AgentV writes to the `default` bucket. The eval file
71-
stays the same across experiments; what changes is the runtime condition.
72-
Dashboards can filter and compare results by experiment.
69+
in `index.jsonl`. CLI `--experiment` wins over `tags.experiment` authored in
70+
the eval file. If neither is set, AgentV writes to the `default` bucket. The
71+
eval file can stay the same across experiments; what changes is the runtime
72+
condition. Dashboards can filter and compare results by experiment.
7373

7474
### Run Specific Test
7575

‎apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,10 @@ agentv eval evals/my-eval.yaml --experiment without_skills
6767
```
6868

6969
The experiment label chooses the result bucket and is propagated to each entry
70-
in `index.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval
71-
file. If neither is set, AgentV writes to the `default` bucket. The eval file
72-
stays the same across experiments; what changes is the runtime condition.
73-
Dashboards can filter and compare results by experiment.
70+
in `index.jsonl`. CLI `--experiment` wins over `tags.experiment` authored in
71+
the eval file. If neither is set, AgentV writes to the `default` bucket. The
72+
eval file can stay the same across experiments; what changes is the runtime
73+
condition. Dashboards can filter and compare results by experiment.
7474

7575
### Run Specific Test
7676

‎docs/adr/0013-stabilize-eval-authoring-contract.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ The preferred eval authoring contract is:
5555

5656
```yaml
5757
name: code-generation-quality
58-
experiment: with-skills
58+
tags:
59+
experiment: with-skills
5960
target: copilot-sdk
6061
evaluate_options:
6162
repeat: 3
@@ -81,12 +82,13 @@ artifact routing, cache keys that should track executable behavior, or result
8182
comparison semantics. Source identity belongs to `eval_path` and run metadata,
8283
not to the display name.
8384

84-
`experiment` remains the optional top-level string run/result grouping label.
85-
It names the condition being measured, such as `baseline`, `candidate`,
85+
The reserved `tags.experiment` key is the eval-authored run/result grouping
86+
label. It names the condition being measured, such as `baseline`, `candidate`,
8687
`with-skills`, or `without-skills`. It is not a runtime-policy object, not a
8788
separate artifact type, and not a storage path namespace. It should not repeat
8889
the suite name, category, target, provider, or model; those are separate
89-
dimensions in run metadata.
90+
dimensions in run metadata. Authored eval YAML does not expose a top-level
91+
`experiment` field.
9092

9193
Top-level `description` is not part of the preferred eval authoring contract.
9294
Existing files that contain it may be read as legacy display metadata, but it

‎packages/core/src/evaluation/loaders/config-loader.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -430,8 +430,10 @@ function stripLocalOnlyExecutionDefaults(
430430
}
431431

432432
function rejectAuthoredRuntimeContainers(suite: JsonObject): void {
433-
if (suite.experiment !== undefined && typeof suite.experiment !== 'string') {
434-
throw new Error("Invalid top-level 'experiment': use a string run/result grouping label.");
433+
if (suite.experiment !== undefined) {
434+
throw new Error(
435+
"Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.",
436+
);
435437
}
436438
if (suite.policy !== undefined) {
437439
throw new Error(

‎packages/core/src/evaluation/validation/eval-file.schema.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -923,6 +923,9 @@ const TagsSchema = z.union([
923923
const TOP_LEVEL_IMPORTS_MESSAGE =
924924
"Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.";
925925

926+
const TOP_LEVEL_EXPERIMENT_MESSAGE =
927+
"Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.";
928+
926929
// ---------------------------------------------------------------------------
927930
// Top-level eval file
928931
// ---------------------------------------------------------------------------
@@ -995,8 +998,8 @@ export const EvalFileSchemaInput: z.ZodType = z.object({
995998
.optional(),
996999
providers: EvalProvidersSchema.optional(),
9971000
model: z.never().optional(),
998-
// Run/result grouping label and flat run controls
999-
experiment: z.string().min(1).optional(),
1001+
// Flat run controls. Group authored YAML with tags.experiment or CLI --experiment.
1002+
experiment: z.never({ invalid_type_error: TOP_LEVEL_EXPERIMENT_MESSAGE }).optional(),
10001003
repeat: z.never().optional(),
10011004
runs: z.never().optional(),
10021005
early_exit: z.never().optional(),

‎packages/core/src/evaluation/validation/eval-validator.ts‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,10 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([
290290

291291
/** Removed top-level fields with migration hints. */
292292
const REMOVED_TOP_LEVEL_FIELDS = new Map<string, string>([
293+
[
294+
'experiment',
295+
"Top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.",
296+
],
293297
[
294298
'imports',
295299
"Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.",
@@ -547,14 +551,6 @@ export async function validateEvalFile(filePath: string): Promise<ValidationResu
547551

548552
// Validate metadata fields
549553
validateMetadata(parsed, absolutePath, errors);
550-
if (parsed.experiment !== undefined && typeof parsed.experiment !== 'string') {
551-
errors.push({
552-
severity: 'error',
553-
filePath: absolutePath,
554-
location: 'experiment',
555-
message: "Top-level 'experiment' must be a string run/result grouping label.",
556-
});
557-
}
558554

559555
// Match promptfoo-style validation: unknown authored suite fields are hard
560556
// errors, not advisory warnings, because they otherwise silently do nothing.

‎packages/core/src/evaluation/yaml-parser.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2621,9 +2621,9 @@ function parentEnvironmentLocation(suite: RawTestSuite): string | undefined {
26212621
}
26222622

26232623
function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonObject | undefined {
2624-
if (suite.experiment !== undefined && typeof suite.experiment !== 'string') {
2624+
if (suite.experiment !== undefined) {
26252625
throw new Error(
2626-
`Invalid eval runtime config in ${evalFilePath}: top-level 'experiment' must be a string run/result grouping label.`,
2626+
`Invalid eval runtime config in ${evalFilePath}: top-level 'experiment' has been removed from authored eval YAML. Use tags.experiment in the eval file or CLI --experiment at run time.`,
26272627
);
26282628
}
26292629
if (suite.policy !== undefined) {
@@ -2689,11 +2689,9 @@ function normalizeSuiteExperimentConfig(parsed: JsonObject): ExperimentConfig |
26892689
readSuiteRuntimeBlock(suite, 'eval file');
26902690
const suiteTargets = extractTargetsFromSuite(parsed);
26912691
const singleSuiteTarget = suiteTargets?.length === 1 ? suiteTargets[0] : undefined;
2692-
const experimentName = asString(suite.experiment);
26932692
const budgetUsd = extractBudgetUsd(parsed);
26942693
const evaluateOptions = isJsonObject(suite.evaluate_options) ? suite.evaluate_options : undefined;
26952694
const runtimeConfig: JsonObject = {
2696-
...(experimentName !== undefined ? { name: experimentName } : {}),
26972695
...(singleSuiteTarget !== undefined ? { target: singleSuiteTarget } : {}),
26982696
...(evaluateOptions?.repeat !== undefined ? { repeat: evaluateOptions.repeat } : {}),
26992697
...(suite.timeout_seconds !== undefined ? { timeout_seconds: suite.timeout_seconds } : {}),

‎packages/core/test/evaluation/eval-inline-experiment.test.ts‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
2323
evalPath,
2424
[
2525
'name: runtime-suite',
26-
'experiment: release-gate',
26+
'tags:',
27+
' experiment: release-gate',
2728
'providers:',
2829
' - id: agentv:codex-cli',
2930
' label: codex',
@@ -49,12 +50,12 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
4950

5051
expect(suite.experimentConfig).toMatchObject({
5152
target: 'codex',
52-
name: 'release-gate',
5353
threshold: 0.7,
5454
repeat: { count: 2, strategy: 'pass_any' },
5555
timeoutSeconds: 30,
5656
budgetUsd: 1.5,
5757
});
58+
expect(suite.tags).toEqual({ experiment: 'release-gate' });
5859
expect(suite.targetSpec).toBeUndefined();
5960
expect(suite.targets).toEqual(['codex']);
6061
});
@@ -667,7 +668,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
667668
await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(/evaluate_options\.repeat/);
668669
});
669670

670-
it('rejects top-level execution blocks and non-string experiment values', async () => {
671+
it('rejects top-level execution blocks and removed experiment values', async () => {
671672
const legacyPath = path.join(tempDir, 'legacy.eval.yaml');
672673
await writeFile(
673674
legacyPath,
@@ -702,8 +703,25 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
702703
].join('\n'),
703704
);
704705

705-
await expect(loadTestSuite(removedPath, tempDir)).rejects.toThrow(
706-
/top-level 'experiment' must be a string/,
706+
await expect(loadTestSuite(removedPath, tempDir)).rejects.toThrow(/tags\.experiment/);
707+
708+
const stringExperimentPath = path.join(tempDir, 'string-experiment.eval.yaml');
709+
await writeFile(
710+
stringExperimentPath,
711+
[
712+
'experiment: release-gate',
713+
'prompts:',
714+
' - "{{ input }}"',
715+
'tests:',
716+
' - id: one',
717+
' criteria: ok',
718+
' vars:',
719+
' input: hello',
720+
].join('\n'),
721+
);
722+
723+
await expect(loadTestSuite(stringExperimentPath, tempDir)).rejects.toThrow(
724+
/tags\.experiment.*CLI --experiment/,
707725
);
708726
});
709727

@@ -1256,7 +1274,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => {
12561274
);
12571275

12581276
await expect(loadTestSuite(parentPath, tempDir)).rejects.toThrow(
1259-
/top-level 'experiment' must be a string/,
1277+
/tags\.experiment.*CLI --experiment/,
12601278
);
12611279
});
12621280

‎packages/core/test/evaluation/validation/eval-file-schema.test.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ describe('EvalFileSchema input shorthand', () => {
293293
const result = EvalFileSchema.safeParse({
294294
name: 'wrapper',
295295
description: 'Wrapper eval',
296-
experiment: 'release-gate',
296+
tags: { experiment: 'release-gate' },
297297
providers: [{ id: 'agentv:codex-cli', label: 'codex' }],
298298
threshold: 0.8,
299299
timeout_seconds: 300,
@@ -337,6 +337,18 @@ describe('EvalFileSchema input shorthand', () => {
337337
expect(result.success).toBe(true);
338338
});
339339

340+
it('rejects top-level experiment strings with a migration hint', () => {
341+
const result = EvalFileSchema.safeParse({
342+
experiment: 'release-gate',
343+
tests: [baseTest],
344+
});
345+
346+
expect(result.success).toBe(false);
347+
expect(result.error?.issues.some((issue) => issue.path.join('.') === 'experiment')).toBe(true);
348+
expect(JSON.stringify(result.error?.issues)).toContain('tags.experiment');
349+
expect(JSON.stringify(result.error?.issues)).toContain('CLI --experiment');
350+
});
351+
340352
it('accepts Promptfoo-style colon provider specs', () => {
341353
const result = EvalFileSchema.safeParse({
342354
name: 'colon-providers',
@@ -790,6 +802,7 @@ describe('EvalFileSchema input shorthand', () => {
790802
});
791803

792804
expect(result.success).toBe(false);
805+
expect(JSON.stringify(result.error?.issues)).toContain('tags.experiment');
793806
});
794807

795808
it('rejects lifecycle commands under authored policy blocks', () => {

‎packages/core/test/evaluation/validation/eval-validator.test.ts‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1066,7 +1066,9 @@ tests:
10661066
(error) =>
10671067
error.severity === 'error' &&
10681068
error.location === 'experiment' &&
1069-
error.message.includes("Top-level 'experiment' must be a string"),
1069+
error.message.includes("Top-level 'experiment' has been removed") &&
1070+
error.message.includes('tags.experiment') &&
1071+
error.message.includes('CLI --experiment'),
10701072
),
10711073
).toBe(true);
10721074
});
@@ -1130,7 +1132,9 @@ tests:
11301132
(error) =>
11311133
error.severity === 'error' &&
11321134
error.location === 'experiment' &&
1133-
error.message.includes("Top-level 'experiment' must be a string"),
1135+
error.message.includes("Top-level 'experiment' has been removed") &&
1136+
error.message.includes('tags.experiment') &&
1137+
error.message.includes('CLI --experiment'),
11341138
),
11351139
).toBe(true);
11361140
});
@@ -1159,7 +1163,9 @@ tests:
11591163
(error) =>
11601164
error.severity === 'error' &&
11611165
error.location === 'experiment' &&
1162-
error.message.includes("Top-level 'experiment' must be a string"),
1166+
error.message.includes("Top-level 'experiment' has been removed") &&
1167+
error.message.includes('tags.experiment') &&
1168+
error.message.includes('CLI --experiment'),
11631169
),
11641170
).toBe(true);
11651171
});
@@ -1187,7 +1193,9 @@ tests:
11871193
(error) =>
11881194
error.severity === 'error' &&
11891195
error.location === 'experiment' &&
1190-
error.message.includes("Top-level 'experiment' must be a string"),
1196+
error.message.includes("Top-level 'experiment' has been removed") &&
1197+
error.message.includes('tags.experiment') &&
1198+
error.message.includes('CLI --experiment'),
11911199
),
11921200
).toBe(true);
11931201
});
@@ -1364,7 +1372,7 @@ tests: []
13641372
expect(importsError?.message).toContain('CLI multi-file selection');
13651373
});
13661374

1367-
it('rejects removed execution blocks when experiment label is present', async () => {
1375+
it('rejects removed execution blocks when top-level experiment is present', async () => {
13681376
const filePath = path.join(tempDir, 'runtime-conflict.yaml');
13691377
await writeFile(
13701378
filePath,
@@ -1381,6 +1389,14 @@ tests:
13811389
const result = await validateEvalFile(filePath);
13821390

13831391
expect(result.valid).toBe(false);
1392+
expect(
1393+
result.errors.some(
1394+
(error) =>
1395+
error.location === 'experiment' &&
1396+
error.message.includes('tags.experiment') &&
1397+
error.message.includes('CLI --experiment'),
1398+
),
1399+
).toBe(true);
13841400
expect(
13851401
result.errors.some(
13861402
(error) =>
@@ -2475,7 +2491,9 @@ tests: "./cases-shorthand-workspace.yaml"
24752491
(error) =>
24762492
error.severity === 'error' &&
24772493
error.location === 'experiment' &&
2478-
error.message.includes("Top-level 'experiment' must be a string"),
2494+
error.message.includes("Top-level 'experiment' has been removed") &&
2495+
error.message.includes('tags.experiment') &&
2496+
error.message.includes('CLI --experiment'),
24792497
),
24802498
).toBe(true);
24812499
});

0 commit comments

Comments
 (0)