fix(providers): emit skill call metadata

Author: christso

apps/web/src/content/docs/docs/next/graders/llm-graders.mdx

@@ -143,8 +143,8 @@ assert:
     prompt: ./prompts/pass-fail.md
 ```
 
-Each `target:` value must match a grader `id` from `.agentv/config.yaml`, an
-eval-local `graders` entry, or a directly referenced `graders: file://...` field.
+Each `target:` value must match a target `id` from `.agentv/config.yaml` or
+from an eval-local `targets` entry.
 
 ### TypeScript Template
 

packages/core/src/evaluation/graders/index.ts

@@ -54,7 +54,6 @@ export type { LlmGraderOptions } from './llm-grader.js';
 
 export { formatToolCalls } from './format-tool-calls.js';
 
-export { SkillTriggerGrader } from './skill-trigger.js';
 export { SkillUsedGrader } from './skill-used.js';
 
 export { assembleLlmGraderPrompt } from './llm-grader-prompt.js';

packages/core/src/evaluation/graders/skill-trigger.ts

@@ -114,7 +114,11 @@ function assertionToNaturalLanguage(entry: RawAssertEntry): string | null {
 
   switch (type) {
     case 'skill-trigger':
-      // Handled separately — not an NL assertion
+      throw new Error(staleSkillTriggerMessage(entry));
+
+    case 'skill-used':
+    case 'not-skill-used':
+      // Handled separately as Agent Skills trigger labels.
       return null;
 
     case 'llm-rubric':
@@ -249,12 +253,46 @@ function assertionToNaturalLanguageList(entry: RawAssertEntry): string[] {
   return nl !== null ? [nl] : [];
 }
 
-/**
- * Extract skill-trigger entries from an assertion list.
- * Returns entries with type === 'skill-trigger'.
- */
-function extractTriggerAssertions(assertions: RawAssertEntry[]): RawAssertEntry[] {
-  return assertions.filter((a) => a.type === 'skill-trigger');
+function staleSkillTriggerMessage(entry: RawAssertEntry): string {
+  const skill = typeof entry.skill === 'string' ? entry.skill.trim() : '';
+  const shouldTrigger = entry.should_trigger !== false;
+  if (!skill) {
+    return "Authored assertion type 'skill-trigger' has been removed. Use 'skill-used' with value: <skill> for expected skill use, or 'not-skill-used' with value: <skill> when the skill must not be used.";
+  }
+  const replacementType = shouldTrigger ? 'skill-used' : 'not-skill-used';
+  return `Authored assertion type 'skill-trigger' has been removed. Replace skill: ${skill} with type: ${replacementType}, value: ${skill}.`;
+}
+
+interface SkillUseAssertion {
+  readonly skill: string;
+  readonly shouldTrigger: boolean;
+}
+
+function skillNameFromValue(value: unknown): string | undefined {
+  if (typeof value === 'string' && value.trim()) {
+    return value.trim();
+  }
+  if (value && typeof value === 'object' && !Array.isArray(value)) {
+    const name = (value as Record<string, unknown>).name;
+    return typeof name === 'string' && name.trim() ? name.trim() : undefined;
+  }
+  return undefined;
+}
+
+function extractSkillUseAssertions(assertions: RawAssertEntry[]): SkillUseAssertion[] {
+  return assertions.flatMap((entry) => {
+    if (entry.type === 'skill-trigger') {
+      throw new Error(staleSkillTriggerMessage(entry));
+    }
+    if (entry.type !== 'skill-used' && entry.type !== 'not-skill-used') {
+      return [];
+    }
+    const skill = skillNameFromValue(entry.value);
+    if (!skill) {
+      return [];
+    }
+    return [{ skill, shouldTrigger: entry.type === 'skill-used' }];
+  });
 }
 
 // ---------------------------------------------------------------------------
@@ -363,9 +401,7 @@ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): Transpi
   const suiteAssertions = rawSuite.assert ?? [];
 
   // Suite-level NL assertions (appended to every test)
-  const suiteNlAssertions: string[] = suiteAssertions
-    .filter((a) => a.type !== 'skill-trigger')
-    .flatMap(assertionToNaturalLanguageList);
+  const suiteNlAssertions: string[] = suiteAssertions.flatMap(assertionToNaturalLanguageList);
 
   /**
    * Helper: get or create the EvalsJsonFile for a skill.
@@ -394,7 +430,7 @@ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): Transpi
       );
     }
 
-    // Collect NL assertions (not skill-trigger)
+    // Collect NL assertions (not skill-use assertions)
     const nlAssertions: string[] = [];
 
     // Prepend test-level criteria as NL assertion
@@ -403,15 +439,15 @@ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): Transpi
     }
 
     for (const entry of caseAssertions) {
-      if (entry.type !== 'skill-trigger') {
+      if (entry.type !== 'skill-used' && entry.type !== 'not-skill-used') {
         nlAssertions.push(...assertionToNaturalLanguageList(entry));
       }
     }
 
     // Append suite-level NL assertions
     nlAssertions.push(...suiteNlAssertions);
 
-    const triggerJudges = extractTriggerAssertions(caseAssertions);
+    const triggerJudges = extractSkillUseAssertions(caseAssertions);
     const { prompt, files: inputFiles } = extractInput(rawCase);
     const expectedOutput = extractExpectedOutput(rawCase.expected_output);
 
@@ -428,7 +464,7 @@ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): Transpi
     };
 
     if (triggerJudges.length === 0) {
-      // No skill-trigger: place in dominant skill (or _no-skill)
+      // No skill-use assertion: place in dominant skill (or _no-skill)
       // Determine dominant skill by scanning all tests (first occurrence wins)
       // We defer this: record with a sentinel and resolve after all tests are processed.
       // For now, push to _no-skill; we'll re-assign at the end.
@@ -437,10 +473,8 @@ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): Transpi
     } else {
       // Place in each skill with the correct should_trigger value
       for (const tj of triggerJudges) {
-        const skillName = typeof tj.skill === 'string' ? tj.skill : '_no-skill';
-        const shouldTrigger = tj.should_trigger !== false; // default true
-        const skillFile = getSkillFile(skillName);
-        skillFile.evals.push({ ...baseCase, should_trigger: shouldTrigger });
+        const skillFile = getSkillFile(tj.skill);
+        skillFile.evals.push({ ...baseCase, should_trigger: tj.shouldTrigger });
       }
     }
   }

packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts

@@ -1390,34 +1390,6 @@ async function parseGraderList(
       continue;
     }
 
-    if (typeValue === 'skill-trigger') {
-      const skillName = asString(rawEvaluator.skill);
-      if (!skillName) {
-        logWarning(`Skipping skill-trigger evaluator '${name}' in '${evalId}': missing skill`);
-        continue;
-      }
-      const rawShouldTrigger = rawEvaluator.should_trigger;
-      const shouldTrigger = typeof rawShouldTrigger === 'boolean' ? rawShouldTrigger : undefined;
-      const weight = validateWeight(rawEvaluator.weight, name, evalId);
-      const { required, min_score } = parseRequiredAndMinScore(
-        rawEvaluator.required,
-        (rawEvaluator as Record<string, unknown>).min_score as JsonValue | undefined,
-        name,
-        evalId,
-      );
-      pushEvaluator({
-        name,
-        type: 'skill-trigger',
-        skill: skillName,
-        ...(shouldTrigger !== undefined ? { should_trigger: shouldTrigger } : {}),
-        ...(weight !== undefined ? { weight } : {}),
-        ...(required !== undefined ? { required } : {}),
-        ...(min_score !== undefined ? { min_score } : {}),
-        ...(negate !== undefined ? { negate } : {}),
-      });
-      continue;
-    }
-
     if (typeValue === 'javascript' || typeValue === 'python' || typeValue === 'webhook') {
       const value = asString(rawEvaluator.value);
       if (!value || value.trim().length === 0) {
@@ -2073,10 +2045,6 @@ function generateAssertionName(typeValue: string, rawEvaluator: JsonObject): str
       }
       return typeValue;
     }
-    case 'skill-trigger': {
-      const skillValue = asString(rawEvaluator.skill);
-      return skillValue ? `skill-trigger-${skillValue}` : 'skill-trigger';
-    }
     case 'contains':
       return value ? `contains-${value}` : 'contains';
     case 'contains-any':

packages/core/src/evaluation/loaders/grader-parser.ts

@@ -13,6 +13,7 @@ import {
   type TargetRuntimeConfig,
   runDockerSandboxCommand,
 } from './sandbox-runner.js';
+import { deriveSkillCallMetadataFromMessages } from './skill-calls.js';
 import { buildTargetExecutionEnvelope, captureTargetExecutionLog } from './target-execution.js';
 import type { CliResolvedConfig } from './targets.js';
 import type {
@@ -543,6 +544,7 @@ export class CliProvider implements Provider {
 
       return {
         output: parsed.output,
+        metadata: deriveSkillCallMetadataFromMessages(parsed.output),
         tokenUsage: parsed.tokenUsage,
         costUsd: parsed.costUsd,
         durationMs: parsed.durationMs ?? measuredDurationMs,
@@ -804,6 +806,7 @@ export class CliProvider implements Provider {
 
         return {
           output: parsed.output,
+          metadata: deriveSkillCallMetadataFromMessages(parsed.output),
           tokenUsage: parsed.tokenUsage,
           costUsd: parsed.costUsd,
           durationMs: parsed.durationMs ?? perRequestFallbackMs,

packages/core/src/evaluation/providers/cli.ts

@@ -23,6 +23,7 @@ import {
 import { resolveDefaultProviderLogDir } from './log-directory.js';
 import { normalizeToolCall } from './normalize-tool-call.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
+import { deriveSkillCallMetadataFromMessages } from './skill-calls.js';
 import { buildTargetExecutionEnvelope } from './target-execution.js';
 import type { CopilotCliResolvedConfig, CopilotCustomProviderConfig } from './targets.js';
 import type {
@@ -369,6 +370,7 @@ export class CopilotCliProvider implements Provider {
           logFile: logger?.filePath,
         },
         output: outputMessages,
+        metadata: deriveSkillCallMetadataFromMessages(outputMessages),
         tokenUsage,
         costUsd,
         durationMs,

packages/core/src/evaluation/providers/copilot-cli.ts

@@ -15,6 +15,7 @@ import {
 import { resolveDefaultProviderLogDir } from './log-directory.js';
 import { normalizeToolCall } from './normalize-tool-call.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
+import { deriveSkillCallMetadataFromMessages } from './skill-calls.js';
 import type { CopilotSdkResolvedConfig } from './targets.js';
 import type {
   Message,
@@ -331,6 +332,7 @@ export class CopilotSdkProvider implements Provider {
           logFile: logger?.filePath,
         },
         output,
+        metadata: deriveSkillCallMetadataFromMessages(output),
         tokenUsage,
         costUsd,
         durationMs,

packages/core/src/evaluation/providers/copilot-sdk.ts

@@ -35,6 +35,7 @@ import {
 } from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
+import { deriveSkillCallMetadataFromMessages } from './skill-calls.js';
 import type { PiCliResolvedConfig } from './targets.js';
 import type {
   Message,
@@ -155,6 +156,7 @@ export class PiCliProvider implements Provider {
           logFile: logger?.filePath,
         },
         output,
+        metadata: deriveSkillCallMetadataFromMessages(output),
         tokenUsage,
         durationMs,
         startTime,
@@ -771,7 +773,7 @@ function extractMessages(events: unknown[]): readonly Message[] {
 
   // Pi CLI may emit tool_execution_start/tool_execution_end events whose tool
   // calls are absent from the final agent_end messages. Reconstruct them and
-  // inject into the last assistant message so evaluators (e.g. skill-trigger)
+  // inject into the last assistant message so trajectory and skill-use graders
   // can detect them.
   const eventToolCalls = extractToolCallsFromEvents(events);
   if (eventToolCalls.length > 0) {

packages/core/src/evaluation/providers/pi-cli.ts

@@ -28,6 +28,7 @@ import {
 } from './pi-provider-aliases.js';
 import { extractPiTextContent, toFiniteNumber, toPiContentArray } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
+import { deriveSkillCallMetadataFromMessages } from './skill-calls.js';
 import type { PiCodingAgentResolvedConfig } from './targets.js';
 import type {
   Message,
@@ -512,6 +513,7 @@ export class PiCodingAgentProvider implements Provider {
             provider: this.config.subprovider,
           },
           output,
+          metadata: deriveSkillCallMetadataFromMessages(output),
           tokenUsage,
           costUsd,
           durationMs,