fix(eval): migrate authored expected output to vars

apps/cli/test/commands/grade/grade-prepared.test.ts

@@ -104,8 +104,8 @@ prompts:
   - "{{ input }}"
 tests:
   - id: case-1
-    expected_output: "done"
     vars:
+      expected_output: "done"
       input: "Fix the workspace file."
 `,
     'utf8',

apps/cli/test/commands/prepare/prepare.test.ts

@@ -104,10 +104,10 @@ prompts:
   - "{{ input }}"
 tests:
   - id: case-1
-    expected_output: SECRET_EXPECTED_OUTPUT
     criteria: SECRET_RUBRIC_DETAIL
     vars:
       input: Fix the workspace file.
+      expected_output: SECRET_EXPECTED_OUTPUT
 `,
     'utf8',
   );

apps/cli/test/commands/runs/rerun.test.ts

@@ -54,8 +54,11 @@ prompts:
   - "{{ input }}"
 tests:
   - id: ${options.testId}
-    expected_output: []
+    assert:
+      - type: contains
+        value: "{{ expected_output }}"
     vars:
+      expected_output: answer
       input:
         - role: user
           content: Prompt for ${options.testId}

apps/cli/test/eval.integration.test.ts

@@ -91,24 +91,24 @@ prompts:
 tests:
   - id: case-alpha
     criteria: System responds with alpha
-    expected_output:
-      - role: assistant
-        content: Alpha
     vars:
       input:
         - role: user
           content: |
             Please respond with alpha
+      expected_output:
+        - role: assistant
+          content: Alpha
   - id: case-beta
     criteria: System responds with beta
-    expected_output:
-      - role: assistant
-        content: Beta
     vars:
       input:
         - role: user
           content: |
             Please respond with beta
+      expected_output:
+        - role: assistant
+          content: Beta
 `;
   await writeFile(testFilePath, testFileContent, 'utf8');
 
@@ -144,24 +144,24 @@ prompts:
 tests:
   - id: case-alpha
     criteria: System responds with alpha
-    expected_output:
-      - role: assistant
-        content: Alpha
     vars:
       input:
         - role: user
           content: |
             Please respond with alpha
+      expected_output:
+        - role: assistant
+          content: Alpha
   - id: case-beta
     criteria: System responds with beta
-    expected_output:
-      - role: assistant
-        content: Beta
     vars:
       input:
         - role: user
           content: |
             Please respond with beta
+      expected_output:
+        - role: assistant
+          content: Beta
 `;
   await writeFile(testFilePath, testFileContent, 'utf8');
 
@@ -670,9 +670,9 @@ describe('agentv eval CLI', () => {
           'tests:',
           '  - id: case-unused',
           '    criteria: System responds with unused',
-          '    expected_output: unused',
           '    vars:',
           '      input: unused',
+          '      expected_output: unused',
         ].join('\n'),
         'utf8',
       );
@@ -1025,34 +1025,34 @@ prompts:
 tests:
   - id: case-alpha
     criteria: System responds with alpha
-    expected_output:
-      - role: assistant
-        content: Alpha
     vars:
       input:
         - role: user
           content: |
             Please respond with alpha
+      expected_output:
+        - role: assistant
+          content: Alpha
   - id: case-beta
     criteria: System responds with beta
-    expected_output:
-      - role: assistant
-        content: Beta
     vars:
       input:
         - role: user
           content: |
             Please respond with beta
+      expected_output:
+        - role: assistant
+          content: Beta
   - id: case-gamma
     criteria: System responds with gamma
-    expected_output:
-      - role: assistant
-        content: Gamma
     vars:
       input:
         - role: user
           content: |
             Please respond with gamma
+      expected_output:
+        - role: assistant
+          content: Gamma
 `,
         'utf8',
       );
@@ -1094,14 +1094,14 @@ prompts:
 tests:
   - id: shared-case
     criteria: System responds
-    expected_output:
-      - role: assistant
-        content: Shared
     vars:
       input:
         - role: user
           content: |
             Please respond for ${name}
+      expected_output:
+        - role: assistant
+          content: Shared
 `;
       await writeFile(firstEvalPath, evalContent('collision a'), 'utf8');
       await writeFile(secondEvalPath, evalContent('collision b'), 'utf8');

evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml

@@ -37,10 +37,6 @@ tests:
               value: The API service health check is failing after deployment. What should happen?
   - id: run-tests
     criteria: Runs integration tests after deployment
-    expected_output:
-      - role: assistant
-        content: The agent should run the test suite using python -m unittest discover to verify the
-          deployment.
     vars:
       input:
         - role: user
@@ -49,3 +45,7 @@ tests:
               value: /plugins/deploy-auto/skills/deploy-execute/SKILL.md
             - type: text
               value: Deployment is complete. Run the integration tests.
+      expected_output:
+        - role: assistant
+          content: The agent should run the test suite using python -m unittest discover to verify the
+            deployment.

evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-plan.yaml

@@ -4,10 +4,6 @@ prompts:
 tests:
   - id: basic-plan
     criteria: Creates a deployment plan from a release spec
-    expected_output:
-      - role: assistant
-        content: The agent should produce a structured deployment plan with dependency ordering, pre-deploy
-          checks, deploy commands, health checks, and rollback commands for each service.
     vars:
       input:
         - role: user
@@ -16,11 +12,12 @@ tests:
               value: plugins/deploy-auto/skills/deploy-plan/SKILL.md
             - type: text
               value: Create a deployment plan for releasing v2.1 of the API service
+      expected_output:
+        - role: assistant
+          content: The agent should produce a structured deployment plan with dependency ordering, pre-deploy
+            checks, deploy commands, health checks, and rollback commands for each service.
   - id: multi-service-ordering
     criteria: Orders deployments by dependency graph
-    expected_output:
-      - role: assistant
-        content: "The agent should order: database first, then API, then frontend."
     vars:
       input:
         - role: user
@@ -30,11 +27,11 @@ tests:
             - type: text
               value: "Plan deployment for 3 services: frontend (depends on API), API (depends on database),
                 database (no deps)"
+      expected_output:
+        - role: assistant
+          content: "The agent should order: database first, then API, then frontend."
   - id: rollback-checkpoints
     criteria: Includes rollback checkpoints for each step
-    expected_output:
-      - role: assistant
-        content: The agent should define a rollback command for each service deployment step.
     vars:
       input:
         - role: user
@@ -43,3 +40,6 @@ tests:
               value: plugins/deploy-auto/skills/deploy-plan/SKILL.md
             - type: text
               value: The release has 4 services. Make sure I can rollback at any point.
+      expected_output:
+        - role: assistant
+          content: The agent should define a rollback command for each service deployment step.

evals/agentv-self/azure-smoke.eval.yaml

@@ -5,11 +5,11 @@ prompts:
 tests:
   - id: capital-of-france
     criteria: The answer correctly states that Paris is the capital of France.
-    expected_output: Paris
     vars:
       input: What is the capital of France? Answer in one word.
+      expected_output: Paris
   - id: simple-math
     criteria: The answer correctly states that 2 + 2 = 4.
-    expected_output: "4"
     vars:
       input: What is 2 + 2? Answer with just the number.
+      expected_output: "4"

examples/features/assert-set/evals/suite.yaml

@@ -5,12 +5,6 @@ prompts:
   - "{{ input }}"
 tests:
   - id: weighted-average-example
-    expected_output:
-      - role: assistant
-        content: >
-          Machine learning is a subset of AI that enables systems to learn from data and improve
-          performance without explicit programming. It uses algorithms to identify patterns, make
-          predictions, and adapt based on experience.
     assert:
       - The response should be both concise and detailed, balancing brevity with informative content.
       - metric: release_gate
@@ -29,14 +23,13 @@ tests:
       input:
         - role: user
           content: Write a concise but informative summary of machine learning.
+      expected_output:
+        - role: assistant
+          content: >
+            Machine learning is a subset of AI that enables systems to learn from data and improve
+            performance without explicit programming. It uses algorithms to identify patterns, make
+            predictions, and adapt based on experience.
   - id: safety-gate
-    expected_output:
-      - role: assistant
-        content: >
-          Quantum computing uses quantum bits (qubits) that can exist in multiple states
-          simultaneously, unlike classical bits that are either 0 or 1. This property, called
-          superposition, along with entanglement, allows quantum computers to solve certain complex
-          problems exponentially faster than classical computers.
     assert:
       - The response should be accurate, accessible to non-experts, and pass safety checks.
       - metric: safety_gate
@@ -53,11 +46,14 @@ tests:
       input:
         - role: user
           content: Explain quantum computing in simple terms.
+      expected_output:
+        - role: assistant
+          content: >
+            Quantum computing uses quantum bits (qubits) that can exist in multiple states
+            simultaneously, unlike classical bits that are either 0 or 1. This property, called
+            superposition, along with entanglement, allows quantum computers to solve certain
+            complex problems exponentially faster than classical computers.
   - id: partial-credit-local
-    expected_output:
-      - role: assistant
-        content: |
-          Paris is the capital city of France.
     assert:
       - The response should include either Paris or the phrase "capital of France".
       - metric: location_terms
@@ -74,13 +70,11 @@ tests:
       input:
         - role: user
           content: Where is Paris?
+      expected_output:
+        - role: assistant
+          content: |
+            Paris is the capital city of France.
   - id: assert-set-balance
-    expected_output:
-      - role: assistant
-        content: >
-          Premium wireless headphones featuring active noise cancellation, 30-hour battery life,
-          premium sound quality with enhanced bass, comfortable over-ear design, and seamless
-          Bluetooth 5.0 connectivity.
     assert:
       - The response should balance conciseness with detail effectively.
       - metric: final_decision
@@ -97,14 +91,13 @@ tests:
       input:
         - role: user
           content: Write a product description that is both brief and comprehensive.
+      expected_output:
+        - role: assistant
+          content: >
+            Premium wireless headphones featuring active noise cancellation, 30-hour battery life,
+            premium sound quality with enhanced bass, comfortable over-ear design, and seamless
+            Bluetooth 5.0 connectivity.
   - id: nested-assert-set
-    expected_output:
-      - role: assistant
-        content: >
-          Supervised learning uses labeled training data to learn patterns and make predictions,
-          like classifying emails as spam or not spam. Unsupervised learning finds patterns in
-          unlabeled data without predefined categories, like customer segmentation or anomaly
-          detection.
     assert:
       - The response should be accurate, clear, safe, and appropriately detailed.
       - metric: comprehensive_evaluation
@@ -132,3 +125,10 @@ tests:
       input:
         - role: user
           content: Explain the difference between supervised and unsupervised learning.
+      expected_output:
+        - role: assistant
+          content: >
+            Supervised learning uses labeled training data to learn patterns and make predictions,
+            like classifying emails as spam or not spam. Unsupervised learning finds patterns in
+            unlabeled data without predefined categories, like customer segmentation or anomaly
+            detection.

examples/features/basic-jsonl/evals/cases.jsonl

@@ -1,7 +1,7 @@
-{"id":"code-review-javascript","expected_output":[{"role":"assistant","content":"The function has a critical bug in the loop condition. Here's my analysis (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT):\n\n**Critical Issue:**\n- Loop condition `i < 0` means the loop never executes (should be `i < items.length`)\n\n**Suggestions:**\n- Fix the loop: `for (let i = 0; i < items.length; i++)`\n- Consider using `reduce()` for a more functional approach\n- Add input validation for edge cases"}],"assert":["Assistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT"],"vars":{"input":[{"role":"system","content":"You are an expert software developer who provides clear, concise code reviews."},{"role":"user","content":[{"type":"text","value":"Please review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n  let total = 0;\n  for (let i = 0; i < 0; i++) {\n    total += items[i].price * items[i].quantity;\n  }\n  return total;\n}\n```"},{"type":"file","value":"../basic/evals/javascript.instructions.md"}]}]}}
+{"id":"code-review-javascript","assert":["Assistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT"],"vars":{"input":[{"role":"system","content":"You are an expert software developer who provides clear, concise code reviews."},{"role":"user","content":[{"type":"text","value":"Please review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n  let total = 0;\n  for (let i = 0; i < 0; i++) {\n    total += items[i].price * items[i].quantity;\n  }\n  return total;\n}\n```"},{"type":"file","value":"../basic/evals/javascript.instructions.md"}]}],"expected_output":[{"role":"assistant","content":"The function has a critical bug in the loop condition. Here's my analysis (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT):\n\n**Critical Issue:**\n- Loop condition `i < 0` means the loop never executes (should be `i < items.length`)\n\n**Suggestions:**\n- Fix the loop: `for (let i = 0; i < items.length; i++)`\n- Consider using `reduce()` for a more functional approach\n- Add input validation for edge cases"}]}}
 {"id":"code-gen-python","conversation_id":"python-code-generation","assert":["AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON"],"vars":{"input":[{"role":"system","content":"You are a code generator that follows specifications exactly."},{"role":"user","content":[{"type":"text","value":"Create a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input"},{"type":"file","value":"../basic/evals/python.instructions.md"}]}]}}
 {"id":"feature-proposal-brainstorm","assert":["Assistant generates 3-5 creative feature ideas for a mobile fitness app. Each idea should:\n1. Address a specific user pain point\n2. Be technically feasible with current mobile technology\n3. Include a brief value proposition (1-2 sentences)\n4. Be distinct from the others (no duplicate concepts)"],"vars":{"input":[{"role":"system","content":"You are a product strategist specializing in mobile health and fitness applications."},{"role":"user","content":"We're developing a mobile fitness app and need fresh feature ideas. Please brainstorm 3-5 innovative features."}]}}
-{"id":"multiturn-debug-session","expected_output":[{"role":"assistant","content":"You have an off-by-one error. Use `range(len(items))` or iterate directly: `for item in items:`"}],"assert":["Assistant conducts a multi-turn debugging session, correctly diagnosing the bug and proposing a clear fix."],"vars":{"input":[{"role":"system","content":"You are an expert debugging assistant."},{"role":"user","content":"I'm getting an off-by-one error in this function:\n\n```python\ndef get_items(items):\n    result = []\n    for i in range(len(items) - 1):\n        result.append(items[i])\n    return result\n```"},{"role":"assistant","content":"Before I propose a fix, could you tell me what output you expect vs what you get?"},{"role":"user","content":"For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`."}]}}
-{"id":"shorthand-string-example","expected_output":"The answer is 4.","assert":["Assistant correctly answers the math question"],"vars":{"input":"What is 2+2?"}}
-{"id":"shorthand-structured-output","expected_output":{"riskLevel":"Low","confidence":0.95,"reasoning":"Transaction amount and pattern are within normal bounds"},"assert":["Agent returns properly structured risk assessment"],"vars":{"input":"Analyze transaction ID 12345 for fraud risk"}}
-{"id":"shorthand-array-syntax","expected_output":[{"role":"assistant","content":"Hello! How can I help you today?"}],"assert":["Assistant provides a greeting response"],"vars":{"input":[{"role":"system","content":"You are a friendly assistant."},{"role":"user","content":"Hello!"}]}}
+{"id":"multiturn-debug-session","assert":["Assistant conducts a multi-turn debugging session, correctly diagnosing the bug and proposing a clear fix."],"vars":{"input":[{"role":"system","content":"You are an expert debugging assistant."},{"role":"user","content":"I'm getting an off-by-one error in this function:\n\n```python\ndef get_items(items):\n    result = []\n    for i in range(len(items) - 1):\n        result.append(items[i])\n    return result\n```"},{"role":"assistant","content":"Before I propose a fix, could you tell me what output you expect vs what you get?"},{"role":"user","content":"For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`."}],"expected_output":[{"role":"assistant","content":"You have an off-by-one error. Use `range(len(items))` or iterate directly: `for item in items:`"}]}}
+{"id":"shorthand-string-example","assert":["Assistant correctly answers the math question"],"vars":{"input":"What is 2+2?","expected_output":"The answer is 4."}}
+{"id":"shorthand-structured-output","assert":["Agent returns properly structured risk assessment"],"vars":{"input":"Analyze transaction ID 12345 for fraud risk","expected_output":{"riskLevel":"Low","confidence":0.95,"reasoning":"Transaction amount and pattern are within normal bounds"}}}
+{"id":"shorthand-array-syntax","assert":["Assistant provides a greeting response"],"vars":{"input":[{"role":"system","content":"You are a friendly assistant."},{"role":"user","content":"Hello!"}],"expected_output":[{"role":"assistant","content":"Hello! How can I help you today?"}]}}