docs(grading): align reference answers and grading artifacts (#1694)

Author: christso

README.md

@@ -110,11 +110,11 @@ tests:
 ```
 
 Plain assertion strings are short-form rubric criteria: AgentV groups them into
-`llm-rubric` and writes each criterion to `grading.json.assertion_results` for the
-Dashboard. Use explicit `type: llm-rubric` when you need weights, required flags, or
-`score_ranges`, or when you need a custom grader prompt, grader target, or
-output transforms; use string `value` for free-form rubric checks. Executable
-graders use `type: script`.
+`llm-rubric` and writes grader detail to `grading.json.component_results` for
+the Dashboard. Use explicit `type: llm-rubric` when you need weights, required
+flags, `score_ranges`, a custom grader prompt, a grader target, or output
+transforms; use string `value` for free-form rubric checks. Executable graders
+use `type: script`.
 
 The target can be an eval-local object when this eval needs target settings of its own:
 
@@ -200,7 +200,7 @@ Run bundle layout:
 │   │   │   └── graders/          #   grader files used
 │   │   └── sample-1/             # one materialized sample
 │   │       ├── result.json       # compact attempt manifest
-│   │       ├── grading.json      # assertion_results and grader evidence
+│   │       ├── grading.json      # pass, score, reason, component_results
 │   │       ├── metrics.json      # tool calls, transcript stats, behavior metrics
 │   │       ├── transcript.json        # normalized agent transcript
 │   │       ├── transcript-raw.jsonl   # raw agent output (debugging)

apps/web/src/content/docs/docs/next/evaluation/batch-cli.mdx

@@ -34,11 +34,11 @@ tests:
   - id: case-001
     criteria: |-
       Batch runner returns JSON with decision=CLEAR.
-
-    expected_output:
-      - role: assistant
-        content:
-          decision: CLEAR
+    vars:
+      expected_output:
+        - role: assistant
+          content:
+            decision: CLEAR
 
     input:
       - role: system
@@ -62,11 +62,11 @@ tests:
   - id: case-002
     criteria: |-
       Batch runner returns JSON with decision=REVIEW.
-
-    expected_output:
-      - role: assistant
-        content:
-          decision: REVIEW
+    vars:
+      expected_output:
+        - role: assistant
+          content:
+            decision: REVIEW
 
     input:
       - role: system
@@ -156,11 +156,12 @@ Each test has its own grader that validates the batch runner output. The grader
 **Output (stdout):**
 ```json
 {
+  "pass": true,
   "score": 1.0,
-  "assertions": [
-    { "text": "decision matches: CLEAR", "passed": true }
-  ],
-  "reasoning": "Batch runner decision matches expected."
+  "reason": "Batch runner decision matches expected.",
+  "checks": [
+    { "text": "decision matches: CLEAR", "pass": true, "reason": "Expected and actual decisions are CLEAR." }
+  ]
 }
 ```
 
@@ -188,22 +189,23 @@ function main() {
     candidateDecision = undefined;
   }
 
-  const assertions: Array<{ text: string; passed: boolean }> = [];
+  const checks: Array<{ text: string; pass: boolean; reason: string }> = [];
 
   if (expectedDecision === candidateDecision) {
-    assertions.push({ text: `decision matches: ${expectedDecision}`, passed: true });
+    checks.push({ text: `decision matches: ${expectedDecision}`, pass: true, reason: 'Expected and actual decisions match.' });
   } else {
-    assertions.push({ text: `mismatch: expected=${expectedDecision} actual=${candidateDecision}`, passed: false });
+    checks.push({ text: `mismatch: expected=${expectedDecision} actual=${candidateDecision}`, pass: false, reason: 'Expected and actual decisions differ.' });
   }
 
-  const passed = assertions.every(a => a.passed);
+  const passed = checks.every((check) => check.pass);
 
   process.stdout.write(JSON.stringify({
+    pass: passed,
     score: passed ? 1 : 0,
-    assertions,
-    reasoning: passed
+    reason: passed
       ? 'Batch runner output matches expected.'
       : 'Batch runner output did not match expected.',
+    checks,
   }));
 }
 
@@ -222,15 +224,16 @@ main();
 
 ## Structured Content
 
-Use structured objects in `expected_output` to define expected output fields for easy validation:
+Use structured objects in `vars.expected_output` to define expected output fields for easy validation:
 
 ```yaml
-expected_output:
-  - role: assistant
-    content:
-      decision: CLEAR
-      confidence: high
-      reasons: []
+vars:
+  expected_output:
+    - role: assistant
+      content:
+        decision: CLEAR
+        confidence: high
+        reasons: []
 ```
 
 The grader extracts these fields and compares them against the parsed candidate output.

apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx

@@ -6,28 +6,34 @@ sidebar:
 slug: docs/evaluation/eval-cases
 ---
 
-Tests are individual test entries within an evaluation file. Each test defines input messages, expected outcomes, and optional grader overrides.
+Tests are individual test entries within an evaluation file. In normal authored
+eval YAML, prompts live in top-level `prompts`, case data lives in
+`tests[].vars`, and graders live in `assert`.
 
 ## Basic Structure
 
 ```yaml
+prompts:
+  - "{{ question }}"
+
 tests:
   - id: addition
-    input: What is 15 + 27?
-
-    expected_output: "42"
+    vars:
+      question: What is 15 + 27?
+      expected_output: "42"
     assert:
-      - The answer is exactly 42
+      - type: llm-rubric
+        value: "Matches the reference answer: {{ expected_output }}"
 ```
 
 ## Fields
 
 | Field | Required | Description |
 |-------|----------|-------------|
 | `id` | Yes | Unique identifier for the test |
-| `input` | Yes | Input sent to the target (string, object, or message array) |
+| `vars` | Yes when the prompt needs row data | Prompt-template variables for this row |
 | `criteria` | No | Optional shared grader guidance for the case |
-| `expected_output` | No | Passive gold/reference data available to graders (string, object, or message array) |
+| `vars.expected_output` | No | Conventional reference-answer var for graders to consume explicitly |
 | `assert` | Yes | Per-test graders; plain strings become `llm-rubric` rubric checks |
 | `execution` | No | Per-case grader/default overrides such as `skip_defaults`; target selection belongs in top-level `target` or CLI `--target` |
 | `environment` | No | Per-case environment recipe (overrides suite-level) |
@@ -78,27 +84,39 @@ names one specific rubric item inside a `llm-rubric` criteria array.
 
 ## Expected Output
 
-Optional reference response for comparison by graders. Write `expected_output`
-as gold/reference data the target could have produced, not as a rubric or "the
-agent should..." criteria list. `expected_output` is passive by default: it is
-stored on the case and passed to graders, but it does not choose a grader by
-itself. A grader may treat it as a strict target, semantic reference, structured
-object, or supporting context depending on the grader type. Add explicit
-assertion strings, `llm-rubric`, `script`, `field-accuracy`, or another
-reference-aware grader when you want the reference data evaluated.
+Optional reference responses belong in `tests[].vars.expected_output` or
+`default_test.vars.expected_output`. Write the value as gold/reference data the
+target could have produced, not as a rubric or "the agent should..." criteria
+list. `expected_output` is passive by default: it is just a var. It does not
+choose a grader by itself. Add explicit assertion strings, `llm-rubric`,
+`script`, `field-accuracy`, or another reference-aware grader when you want the
+reference data evaluated.
 
-A string expands to a single assistant message:
+A low-friction Promptfoo-compatible semantic check puts the reference in vars
+and consumes it from an explicit `llm-rubric` assertion:
 
 ```yaml
-expected_output: "42"
+prompts:
+  - "{{ question }}"
+
+tests:
+  - id: addition
+    vars:
+      question: What is 15 + 27?
+      expected_output: "42"
+    assert:
+      - type: llm-rubric
+        value: "Matches the reference answer: {{ expected_output }}"
 ```
 
-For structured or multi-message expected output, use a message array:
+For structured expected output, keep the object under `vars.expected_output`
+and use a grader that knows how to compare it:
 
 ```yaml
-expected_output:
-  - role: assistant
-    content: "42"
+vars:
+  expected_output:
+    status: approved
+    confidence: 0.95
 ```
 
 ## Per-Case Execution Overrides
@@ -354,12 +372,9 @@ context; each rubric item uses `outcome` for the specific desired behavior being
 ```yaml
 tests:
   - id: denied-party
-    input:
-      - role: user
-        content: Screen "Acme Corp" against denied parties list
-    expected_output:
-      - role: assistant
-        content: "DENIED"
+    vars:
+      input: Screen "Acme Corp" against denied parties list
+      expected_output: "DENIED"
     assert:
       - type: contains
         value: "DENIED"
@@ -409,11 +424,11 @@ Required gates are evaluated after all graders run. If any required grader falls
 
 ## How Reference Fields and `assert` Interact
 
-`expected_output` is reference data, not a grader. It is stored on the case and
-provided to graders that know how to use it, but it does not create an LLM
-grading call by itself. A grader can use that data as an exact target, a
-semantic reference, a structured comparison object, or supporting context. Put
-the grading contract in `assert`.
+`vars.expected_output` is reference data, not a grader. It is provided to
+graders that know how to use it, but it does not create an LLM grading call by
+itself. A grader can use that data as an exact target, a semantic reference, a
+structured comparison object, or supporting context. Put the grading contract in
+`assert`.
 
 Plain assertion strings are the default shape for semantic checks:
 
@@ -442,18 +457,19 @@ tests:
 ```
 
 When `assert` is defined, only the declared graders run. No implicit grader is
-added because `expected_output` exists. Declared graders such as plain rubric
-strings, explicit `llm-rubric` entries, or `script` graders receive the case context, including
-`expected_output`, as input automatically.
+added because `vars.expected_output` exists. Declared graders such as plain
+rubric strings, explicit `llm-rubric` entries, or `script` graders receive the
+case context, including `expected_output`, as input automatically.
 
-This means a case with `expected_output` and only deterministic assertions evaluates only
-those deterministic assertions:
+This means a case with `vars.expected_output` and only deterministic assertions
+evaluates only those deterministic assertions:
 
 ```yaml
 tests:
   - id: deterministic-reference
-    input: "What is 2 + 2?"
-    expected_output: "4"        # reference data only
+    vars:
+      input: "What is 2 + 2?"
+      expected_output: "4"      # reference data only
     assert:
       - type: contains          # only this grader runs
         value: "4"
@@ -465,12 +481,13 @@ keep those checks in `assert`:
 ```yaml
 tests:
   - id: verification-learning-capture
-    input: |
-      Decide what durable repo change should be made after a PR closeout
-      revealed reusable verification workflow lessons.
-    expected_output: |
-      The durable repo change is to update .agents/verification.md with the
-      reusable verification workflow lessons.
+    vars:
+      input: |
+        Decide what durable repo change should be made after a PR closeout
+        revealed reusable verification workflow lessons.
+      expected_output: |
+        The durable repo change is to update .agents/verification.md with the
+        reusable verification workflow lessons.
     assert:
       - The answer recommends updating .agents/verification.md rather than leaving the learning only in PR comments or private evidence.
       - The answer avoids preserving one-off observations as durable guidance.
@@ -527,4 +544,4 @@ tests:
 `metadata` is passed to lifecycle hooks as `case_metadata`, preserved
 in result records, and available to in-process custom assertions. AgentV does
 not interpret arbitrary metadata keys itself; use `environment`, `extensions`,
-`prompts`, `expected_output`, and `assert` for operational behavior.
+`prompts`, `vars.expected_output`, and `assert` for operational behavior.

apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx

@@ -105,8 +105,9 @@ Raw cases are just case data:
 ```yaml
 # evals/cases/refund-smoke.cases.yaml
 - id: damaged-item
-  input: The item arrived damaged. What should support do?
-  expected_output: Offer a replacement or refund path.
+  vars:
+    question: The item arrived damaged. What should support do?
+    expected_output: Offer a replacement or refund path.
 ```
 
 A run-focused eval stays ordinary eval YAML while choosing a target and run controls:
@@ -126,7 +127,10 @@ tests:
   - id: local-edge-case
     vars:
       question: Can a final-sale item be refunded after damage in transit?
-    expected_output: Explain the final-sale exception for damaged transit.
+      expected_output: Explain the final-sale exception for damaged transit.
+    assert:
+      - type: llm-rubric
+        value: "Matches the reference answer: {{ expected_output }}"
 ```
 
 The `experiments/` directory is optional and user-owned. AgentV does not infer
@@ -156,7 +160,10 @@ tests:
   - id: addition
     vars:
       question: What is 15 + 27?
-    expected_output: "42"
+      expected_output: "42"
+    assert:
+      - type: llm-rubric
+        value: "Matches the reference answer: {{ expected_output }}"
 ```
 
 ### Top-level Fields
@@ -214,6 +221,9 @@ prompts:
 default_test:
   vars:
     audience: engineers
+  assert:
+    - type: llm-rubric
+      value: "Matches the reference answer: {{ expected_output }}"
 
 targets:
   - id: local-mini
@@ -232,15 +242,15 @@ tests:
   - id: release-notes
     vars:
       topic: the July release notes
-    expected_output: concise release-note summary
+      expected_output: concise release-note summary
     assert:
       - Identifies the most important change
       - Avoids unsupported details
   - id: roadmap
     vars:
       audience: executives
       topic: the next roadmap phase
-    expected_output: concise roadmap summary
+      expected_output: concise roadmap summary
     assert:
       - Identifies the main product direction
       - Uses the requested audience framing
@@ -662,16 +672,18 @@ tests:
   - id: capital
     vars:
       question: What is the capital of France?
-      expected_answer: Paris
+      expected_output: Paris
     criteria: "Answers {{ vars.question }} correctly"
-    expected_output: "{{ vars.expected_answer }}"
+    assert:
+      - type: llm-rubric
+        value: "Matches the reference answer: {{ expected_output }}"
 ```
 
 ### Behavior
 
 - `vars` is defined per test as an object, with optional defaults from `default_test.vars`
 - `{{ vars.name }}` and dotted paths like `{{ vars.user.name }}` are supported
-- Substitution applies to `prompts`, `criteria`, `expected_output`, assertion values/metrics, and conversation turn `input` / `expected_output` / assertions
+- Substitution applies to `prompts`, `criteria`, `vars.expected_output`, assertion values/metrics, and conversation turn `input` / `expected_output` / assertions
 - When the whole string is a single placeholder, the original JSON value is preserved
 - Missing variables render as empty strings following Nunjucks semantics
 - `vars` interpolation is separate from environment interpolation: `{{ vars.question }}` uses test data, `{{ env.PROJECT_NAME }}` uses environment variables