fix(schema): reject stale authored skill and trajectory graders (#1686)

* fix(schema): reject stale authored skill and trajectory graders

* test(cli): migrate prepared grade trajectory assertions

* fix(codemod): migrate stale snake case grader aliases

* docs(examples): retire stale grader example names

Author: christso

apps/cli/test/commands/grade/grade-prepared.test.ts

@@ -254,12 +254,18 @@ describe('agentv grade prepared attempts', () => {
       tempDir,
       `
 - metric: expected-tool-sequence
-  type: tool-trajectory
-  mode: exact
-  expected:
-    - tool: Read
-      args:
-        path: app.txt
+  type: trajectory:tool-sequence
+  value:
+    mode: exact
+    steps:
+      - Read
+- metric: expected-tool-args
+  type: trajectory:tool-args-match
+  value:
+    name: Read
+    args:
+      path: app.txt
+    mode: partial
 `,
     );
     const preparedDir = path.join(tempDir, 'prepared', 'trace-tools');
@@ -287,10 +293,17 @@ describe('agentv grade prepared attempts', () => {
     expect(row.score).toBe(0);
     expect(row.scores[0]).toMatchObject({
       name: 'expected-tool-sequence',
-      type: 'tool-trajectory',
+      type: 'trajectory:tool-sequence',
       score: 0,
       pass: false,
-      reason: 'No trace available for evaluation',
+      reason: 'No trace data available for trajectory:tool-sequence assertion',
+    });
+    expect(row.scores[1]).toMatchObject({
+      name: 'expected-tool-args',
+      type: 'trajectory:tool-args-match',
+      score: 0,
+      pass: false,
+      reason: 'No trace data available for trajectory:tool-args-match assertion',
     });
   });
 
@@ -299,12 +312,18 @@ describe('agentv grade prepared attempts', () => {
       tempDir,
       `
 - metric: expected-tool-sequence
-  type: tool-trajectory
-  mode: in_order
-  expected:
-    - tool: Read
-      args:
-        path: app.txt
+  type: trajectory:tool-sequence
+  value:
+    mode: in_order
+    steps:
+      - Read
+- metric: expected-tool-args
+  type: trajectory:tool-args-match
+  value:
+    name: Read
+    args:
+      path: app.txt
+    mode: partial
 `,
     );
     const preparedDir = path.join(tempDir, 'prepared', 'trace-tools-with-trace');
@@ -397,7 +416,14 @@ describe('agentv grade prepared attempts', () => {
     expect(row.score).toBe(1);
     expect(row.scores[0]).toMatchObject({
       name: 'expected-tool-sequence',
-      type: 'tool-trajectory',
+      type: 'trajectory:tool-sequence',
+      score: 1,
+      pass: true,
+      reason: 'Grader passed.',
+    });
+    expect(row.scores[1]).toMatchObject({
+      name: 'expected-tool-args',
+      type: 'trajectory:tool-args-match',
       score: 1,
       pass: true,
       reason: 'Grader passed.',
@@ -410,10 +436,11 @@ describe('agentv grade prepared attempts', () => {
       tempDir,
       `
 - metric: expected-tool-sequence
-  type: tool-trajectory
-  mode: in_order
-  expected:
-    - tool: Read
+  type: trajectory:tool-sequence
+  value:
+    mode: in_order
+    steps:
+      - Read
 `,
     );
     const preparedDir = path.join(tempDir, 'prepared', 'wrong-transcript-target');
@@ -453,10 +480,11 @@ describe('agentv grade prepared attempts', () => {
       tempDir,
       `
 - metric: expected-tool-sequence
-  type: tool-trajectory
-  mode: in_order
-  expected:
-    - tool: Read
+  type: trajectory:tool-sequence
+  value:
+    mode: in_order
+    steps:
+      - Read
 `,
     );
     const preparedDir = path.join(tempDir, 'prepared', 'wrong-envelope-target');

docs/plans/2026-06-06-001-agentv-eval-authoring-extensibility-plan.md

@@ -268,7 +268,7 @@ Initial reference evals to consider:
 
 - WTG.AI.Prompts PR 679 as the primary WTG example, because it represents skill/prompt QA, workspace-sensitive coding-agent behavior, regression checks, pinned workspace setup, and before/after eval-driven prompt improvement in one artifact set.
 - `financial-research-agent/evals/financial-research-agent.eval.yaml` for Dexter-derived financial research rubric/provenance/operator semantics.
-- AgentV public examples such as `examples/features/rubric/`, `examples/showcase/grader-conformance/`, `examples/features/trace-evaluation/`, and `examples/features/tool-trajectory-simple/` are secondary reference material, not the main private parity corpus.
+- AgentV public examples such as `examples/features/rubric/`, `examples/showcase/grader-conformance/`, `examples/features/trace-evaluation/`, and `examples/features/trajectory-assertions-simple/` are secondary reference material, not the main private parity corpus.
 
 Placement constraints:
 

docs/plans/public-agentv-demo-projects.md

@@ -276,7 +276,7 @@ Out of scope:
 - Origin requirements: `docs/brainstorms/2026-06-04-public-agentv-demo-projects-requirements.md`
 - Existing plugin-variant pattern: `examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml`, `examples/showcase/bug-fix-benchmark/scripts/setup-variant.sh`
 - Existing previous-commit workspace pattern: `examples/features/repo-lifecycle/evals/suite.yaml`, `examples/showcase/cross-repo-sync/evals/suite.yaml`, `examples/showcase/cross-repo-sync/scripts/setup.ts`
-- Existing multi-provider skill eval pattern: `examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml`
+- Existing multi-provider skill eval pattern: `examples/features/agent-skills-evals/multi-provider-skill-use.EVAL.yaml`
 - Project registry and wire format: `packages/core/src/projects.ts`, `packages/core/test/projects.test.ts`
 - Existing private results-repo wiring: `agentv-deploy` repository setup scripts and result config blocks
 - Git checkout behavior: `packages/core/src/evaluation/workspace/repo-manager.ts`, `packages/core/test/evaluation/workspace/repo-manager.test.ts`

evals/agentic-engineering/agent-plugin-review.eval.yaml

@@ -15,8 +15,8 @@ tests:
   - id: detect-missing-eval
     criteria: Identifies that deploy-rollback skill has no corresponding eval file
     assert:
-      - type: skill-trigger
-        skill: agent-plugin-review
+      - type: skill-used
+        value: agent-plugin-review
       - type: contains
         value: deploy-rollback
       - type: rubrics

examples/README.md

@@ -42,8 +42,8 @@ Focused demonstrations of specific AgentV capabilities. Each example includes it
 
 - [basic](features/basic/) - Core schema features
 - [rubric](features/rubric/) - Rubric-based evaluation
-- [tool-trajectory-simple](features/tool-trajectory-simple/) - Tool trajectory validation
-- [tool-trajectory-advanced](features/tool-trajectory-advanced/) - Advanced tool trajectory with expected_output
+- [trajectory-assertions-simple](features/trajectory-assertions-simple/) - Promptfoo trajectory assertion validation
+- [trajectory-assertions-advanced](features/trajectory-assertions-advanced/) - Advanced trajectory assertions with expected_output
 - [assert-set](features/assert-set/) - Assertion grouping patterns
 - [weighted-graders](features/weighted-graders/) - Weighted graders
 - [execution-metrics](features/execution-metrics/) - Metrics tracking (tokens, cost, latency)

examples/features/README.md

@@ -49,9 +49,9 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 ### Evaluate tool use and agent behavior
 | Example | Description |
 |---------|-------------|
-| [tool-trajectory-simple](tool-trajectory-simple/) | Validate expected tool call sequences |
-| [tool-trajectory-advanced](tool-trajectory-advanced/) | Tool trajectory checks with `expected_output` and per-call assertions |
-| [latency-assertions](latency-assertions/) | Assert `max_duration_ms` per tool call to catch performance regressions |
+| [trajectory-assertions-simple](trajectory-assertions-simple/) | Validate expected tool calls with Promptfoo trajectory assertions |
+| [trajectory-assertions-advanced](trajectory-assertions-advanced/) | Promptfoo trajectory assertions with `expected_output` and per-call checks |
+| [latency-assertions](latency-assertions/) | Tool sequence and argument checks for a latency-flavored mock target |
 | [tool-evaluation-plugins](tool-evaluation-plugins/) | F1 precision/recall scoring for tool-call accuracy |
 | [trace-evaluation](trace-evaluation/) | Inspect agent internals: LLM call counts, tool executions, step durations |
 
@@ -172,8 +172,8 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 | [test-vars-templating](test-vars-templating/) | Dataset & prompt templates |
 | [threshold-grader](threshold-grader/) | LLM grading |
 | [tool-evaluation-plugins](tool-evaluation-plugins/) | Tool & agent evaluation |
-| [tool-trajectory-advanced](tool-trajectory-advanced/) | Tool & agent evaluation |
-| [tool-trajectory-simple](tool-trajectory-simple/) | Tool & agent evaluation |
+| [trajectory-assertions-advanced](trajectory-assertions-advanced/) | Tool & agent evaluation |
+| [trajectory-assertions-simple](trajectory-assertions-simple/) | Tool & agent evaluation |
 | [trace-analysis](trace-analysis/) | Offline evaluation |
 | [trace-evaluation](trace-evaluation/) | Tool & agent evaluation |
 | [trial-output-consistency](trial-output-consistency/) | Benchmarking |

examples/features/agent-skills-evals/README.md

@@ -63,9 +63,8 @@ tests:
           - type: text
             value: "Find the top 3 months by revenue."
     assert:
-      - type: skill-trigger
-        skill: csv-analyzer
-        should_trigger: true
+      - type: skill-used
+        value: csv-analyzer
       - type: llm-rubric
         value: "Output identifies November as the highest revenue month"
       - type: contains
@@ -98,14 +97,14 @@ The `csv-analyzer` skill is included in this example under `.claude/skills/csv-a
 
 ## Multi-provider eval
 
-`multi-provider-skill-trigger.EVAL.yaml` tests the same skill across multiple providers:
+`multi-provider-skill-use.EVAL.yaml` tests the same skill across multiple providers:
 
 ```bash
-bun apps/cli/src/cli.ts eval multi-provider-skill-trigger.EVAL.yaml \
+bun apps/cli/src/cli.ts eval multi-provider-skill-use.EVAL.yaml \
   --target copilot-cli --targets ../.agentv/targets.yaml
 ```
 
-The `skill-trigger` grader automatically handles each provider's tool-call format:
+The `skill-used` and `not-skill-used` graders automatically handle each provider's tool-call format:
 
 | Provider | Detection method |
 |----------|-----------------|
@@ -115,7 +114,7 @@ The `skill-trigger` grader automatically handles each provider's tool-call forma
 
 ## Copilot note
 
-When running `skill-trigger` evals against Copilot targets, real traces may show provider-specific tool names such as:
+When running skill-use evals against Copilot targets, real traces may show provider-specific tool names such as:
 
 ```text
 Using skill: <skill-name>

examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml

@@ -1,6 +1,6 @@
 tags:
   - agent
-  - skill-trigger
+  - skill-use
 extensions:
   - agentv:agent-rules
 environment:
@@ -11,9 +11,8 @@ prompts:
 tests:
   - id: csv-top-months
     assert:
-      - type: skill-trigger
-        skill: csv-analyzer
-        should_trigger: true
+      - type: skill-used
+        value: csv-analyzer
       - type: llm-rubric
         value:
           - "Output applies seasonal weighting factors (Q1: 0.85, Q2: 1.00, Q3: 1.15, Q4: 1.25)"
@@ -35,8 +34,7 @@ tests:
                 sure to apply the seasonal weighting formula from the skill.
   - id: irrelevant-query
     assert:
-      - type: skill-trigger
-        skill: csv-analyzer
-        should_trigger: false
+      - type: not-skill-used
+        value: csv-analyzer
     vars:
       input: What time is it?

…s/multi-provider-skill-trigger.EVAL.yaml …evals/multi-provider-skill-use.EVAL.yamlexamples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml renamed to examples/features/agent-skills-evals/multi-provider-skill-use.EVAL.yaml examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml renamed to examples/features/agent-skills-evals/multi-provider-skill-use.EVAL.yaml

@@ -8,29 +8,25 @@ prompts:
 tests:
   - id: should-trigger-direct-request
     assert:
-      - type: skill-trigger
-        skill: acme-deploy
-        should_trigger: true
+      - type: skill-used
+        value: acme-deploy
     vars:
       input: How do I deploy payments-api to production?
   - id: should-trigger-casual-phrasing
     assert:
-      - type: skill-trigger
-        skill: acme-deploy
-        should_trigger: true
+      - type: skill-used
+        value: acme-deploy
     vars:
       input: I need to roll back user-service in staging, what's the Acme deploy procedure for that?
   - id: should-not-trigger-unrelated
     assert:
-      - type: skill-trigger
-        skill: acme-deploy
-        should_trigger: false
+      - type: not-skill-used
+        value: acme-deploy
     vars:
       input: What time is it in Tokyo right now?
   - id: should-not-trigger-near-miss
     assert:
-      - type: skill-trigger
-        skill: acme-deploy
-        should_trigger: false
+      - type: not-skill-used
+        value: acme-deploy
     vars:
       input: Write a Python function that parses JSON logs and extracts error messages

examples/features/batch-cli/README.md

@@ -10,7 +10,7 @@ This example demonstrates an **external batch runner** pattern for a (synthetic)
 
 3. **Batch processing**: `batch-cli-runner.ts` reads the CSV and applies synthetic AML screening rules, writing **actual responses** as JSONL to a temporary file. Each JSONL record includes `output` with `tool_calls` for trace extraction.
 
-4. **Evaluation**: AgentV compares the actual JSONL output against the ground truth in `evals/suite.yaml` using graders like `script` and `tool-trajectory`.
+4. **Evaluation**: AgentV compares the actual JSONL output against the ground truth in `evals/suite.yaml` using graders like `script` and `trajectory:tool-used`.
 
 ## Batch error handling (missing JSONL id)
 
@@ -45,7 +45,7 @@ The batch runner outputs JSONL records with `output` containing `tool_calls`:
 }
 ```
 
-The `tool_trajectory` grader extracts tool calls directly from `output[].tool_calls[]`. This is the primary format - no separate `trace` field is required.
+Trajectory assertions extract tool calls directly from `output[].tool_calls[]`. This is the primary format - no separate `trace` field is required.
 
 ## Files