feat(evals): preserve rubric criterion operators

Bead: av-r0s.1

Author: christso

apps/web/src/content/docs/docs/evaluation/rubrics.mdx

@@ -67,6 +67,7 @@ assertions:
 |-------|---------|-------------|
 | `id` | Auto-generated | Unique identifier for the criterion |
 | `outcome` | — | Description of what to check |
+| `operator` | — | Optional intent hint: `correctness` or `contradiction` |
 | `weight` | `1.0` | Relative importance for scoring |
 | `required` | `false` | If true, failing this criterion fails the entire eval |
 | `min_score` | — | Minimum score (0–1) for this criterion to pass |
@@ -76,6 +77,26 @@ assertions:
 `required_min_score` (0–10 integer scale) is deprecated. Use `min_score` (0–1 scale) instead. For example, `required_min_score: 8` becomes `min_score: 0.8`.
 :::
 
+### Criterion Operators
+
+Use `operator` when the criterion outcome should be interpreted with a specific grading intent instead of relying on the wording in `outcome`.
+
+```yaml
+assertions:
+  - type: rubrics
+    criteria:
+      - id: supported-revenue
+        operator: correctness
+        outcome: States revenue increased to $10M
+        required: true
+      - id: no-revenue-conflict
+        operator: contradiction
+        outcome: Revenue increased to $10M
+        required: true
+```
+
+`correctness` requires the answer to positively satisfy the outcome. `contradiction` is a guard: the answer passes when it does not make an incompatible claim, even if it omits the outcome entirely.
+
 ## Score-Range Mode (Analytic)
 
 For quality gradients instead of binary pass/fail, use score ranges:

examples/features/rubric/README.md

@@ -7,6 +7,7 @@ Demonstrates rubric-based evaluation with weights, required flags, and auto-gene
 - Inline rubrics as strings
 - Rubric objects with weights
 - Required vs optional criteria
+- Criterion operators for correctness and contradiction guards
 - Auto-generating rubrics from criteria
 - Rubric file references
 
@@ -20,4 +21,5 @@ bun agentv eval examples/features/rubric/evals/dataset.eval.yaml --target defaul
 ## Key Files
 
 - `evals/dataset.eval.yaml` - Test cases with various rubric patterns
+- `evals/operators.eval.yaml` - Focused example of correctness and contradiction operators
 - `evals/rubrics/` - External rubric files

examples/features/rubric/evals/operators.eval.yaml

@@ -0,0 +1,42 @@
+# AgentV Rubric Operator Example
+# Demonstrates optional criterion operators for preserving rubric intent.
+
+name: rubric-operators
+description: "Focused example showing correctness and contradiction rubric operators"
+
+execution:
+  target: llm
+
+tests:
+  - id: finance-summary-operator-guards
+    criteria: |-
+      Summarize the source faithfully. Include supported material facts and avoid
+      claims that contradict the source.
+
+    input:
+      - role: user
+        content: |-
+          Source note:
+          - Q4 revenue increased to $10M from $8M.
+          - Gross margin declined to 42% from 45%.
+
+          Write a two-sentence finance summary.
+
+    expected_output:
+      - role: assistant
+        content: |-
+          Q4 revenue increased to $10M from $8M. Gross margin declined to 42%
+          from 45%.
+
+    assertions:
+      - type: rubrics
+        criteria:
+          - id: supported-revenue
+            operator: correctness
+            outcome: States Q4 revenue increased to $10M from $8M.
+            required: true
+
+          - id: no-margin-contradiction
+            operator: contradiction
+            outcome: Gross margin declined to 42% from 45%.
+            required: true

packages/core/src/evaluation/graders/llm-grader-prompt.ts

@@ -9,6 +9,7 @@ import {
   buildScoreRangeOutputSchema,
   substituteVariables,
 } from './llm-grader.js';
+import { formatRubricOperatorGuidance, formatRubricOperatorLabel } from './rubric-operators.js';
 
 export interface LlmGraderPromptAssembly {
   systemPrompt: string;
@@ -144,10 +145,20 @@ function assembleChecklist(
 
   parts.push('[[ ## rubrics ## ]]');
 
+  const operatorGuidance = formatRubricOperatorGuidance(rubrics);
+  if (operatorGuidance.length > 0) {
+    parts.push('', 'Operator guidance:');
+    for (const guidance of operatorGuidance) {
+      parts.push(`- ${guidance}`);
+    }
+    parts.push('');
+  }
+
   for (const rubric of rubrics) {
     const requiredLabel = rubric.required ? ' (REQUIRED)' : '';
     const weightLabel = rubric.weight !== 1.0 ? ` (weight: ${rubric.weight})` : '';
-    parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.outcome}`);
+    const operatorLabel = formatRubricOperatorLabel(rubric.operator);
+    parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}${operatorLabel}: ${rubric.outcome}`);
   }
 
   parts.push('', 'For each rubric, determine if it is satisfied and provide brief reasoning.');
@@ -213,6 +224,10 @@ function assembleScoreRange(
 
     parts.push('', `### Criterion: ${rubric.id}${weightLabel}${minScoreLabel}`);
 
+    if (rubric.operator) {
+      parts.push(`Operator: ${rubric.operator}`);
+    }
+
     if (rubric.outcome) {
       parts.push(`Description: ${rubric.outcome}`);
     }
@@ -227,6 +242,11 @@ function assembleScoreRange(
     }
   }
 
+  const operatorGuidance = formatRubricOperatorGuidance(rubrics);
+  if (operatorGuidance.length > 0) {
+    parts.push('', ...operatorGuidance);
+  }
+
   parts.push(
     '',
     'For each criterion, provide an integer score 0-10 that matches one of its defined score ranges.',

packages/core/src/evaluation/graders/llm-grader.ts

@@ -14,6 +14,7 @@ import { extractLastAssistantContent, isAgentProvider } from '../providers/types
 import { DEPRECATED_TEMPLATE_VARIABLES, TEMPLATE_VARIABLES } from '../template-variables.js';
 import type { TokenUsage } from '../trace.js';
 import type { AssertionEntry, JsonObject, RubricItem } from '../types.js';
+import { formatRubricOperatorGuidance, formatRubricOperatorLabel } from './rubric-operators.js';
 import { clampScore, isNonEmptyString, parseJsonFromText, scoreToVerdict } from './scoring.js';
 import type { EvaluationContext, EvaluationScore, Grader } from './types.js';
 
@@ -952,6 +953,10 @@ export class LlmGrader implements Grader {
 
       parts.push('', `### Criterion: ${rubric.id}${weightLabel}${minScoreLabel}`);
 
+      if (rubric.operator) {
+        parts.push(`Operator: ${rubric.operator}`);
+      }
+
       if (rubric.outcome) {
         parts.push(`Description: ${rubric.outcome}`);
       }
@@ -966,6 +971,11 @@ export class LlmGrader implements Grader {
       }
     }
 
+    const operatorGuidance = formatRubricOperatorGuidance(rubrics);
+    if (operatorGuidance.length > 0) {
+      parts.push('', ...operatorGuidance);
+    }
+
     parts.push(
       '',
       'For each criterion, provide an integer score 0-10 that matches one of its defined score ranges.',
@@ -1007,10 +1017,22 @@ export class LlmGrader implements Grader {
 
     parts.push('[[ ## rubrics ## ]]');
 
+    const operatorGuidance = formatRubricOperatorGuidance(rubrics);
+    if (operatorGuidance.length > 0) {
+      parts.push('', 'Operator guidance:');
+      for (const guidance of operatorGuidance) {
+        parts.push(`- ${guidance}`);
+      }
+      parts.push('');
+    }
+
     for (const rubric of rubrics) {
       const requiredLabel = rubric.required ? ' (REQUIRED)' : '';
       const weightLabel = rubric.weight !== 1.0 ? ` (weight: ${rubric.weight})` : '';
-      parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.outcome}`);
+      const operatorLabel = formatRubricOperatorLabel(rubric.operator);
+      parts.push(
+        `- [${rubric.id}]${requiredLabel}${weightLabel}${operatorLabel}: ${rubric.outcome}`,
+      );
     }
 
     parts.push('', 'For each rubric, determine if it is satisfied and provide brief reasoning.');

packages/core/src/evaluation/graders/rubric-operators.ts

@@ -0,0 +1,27 @@
+import type { RubricItem, RubricOperator } from '../types.js';
+
+const OPERATOR_GUIDANCE: Record<RubricOperator, string> = {
+  correctness:
+    'Correctness: mark satisfied only when the answer positively supports or fulfills the outcome. Omission or contradiction should not satisfy it.',
+  contradiction:
+    'Contradiction guard: mark satisfied when the answer does not make a claim that contradicts the outcome. Do not require the answer to mention the outcome; mark unsatisfied only for incompatible claims.',
+};
+
+export function formatRubricOperatorLabel(operator: RubricOperator | undefined): string {
+  return operator ? ` (operator: ${operator})` : '';
+}
+
+export function formatRubricOperatorGuidance(rubrics: readonly RubricItem[]): readonly string[] {
+  const operators = new Set<RubricOperator>();
+  for (const rubric of rubrics) {
+    if (rubric.operator) {
+      operators.add(rubric.operator);
+    }
+  }
+
+  if (operators.size === 0) {
+    return [];
+  }
+
+  return [...operators].map((operator) => OPERATOR_GUIDANCE[operator]);
+}

packages/core/src/evaluation/loaders/grader-parser.ts

@@ -10,8 +10,9 @@ import type {
   GraderKind,
   JsonObject,
   JsonValue,
+  RubricOperator,
 } from '../types.js';
-import { isGraderKind } from '../types.js';
+import { RUBRIC_OPERATOR_VALUES, isGraderKind } from '../types.js';
 import { validateCustomPromptContent } from '../validation/prompt-validator.js';
 import { parseYamlValue } from '../yaml-loader.js';
 import { resolveFileReference } from './file-resolver.js';
@@ -1940,6 +1941,28 @@ function isValidFieldAggregationType(
   return typeof value === 'string' && VALID_FIELD_AGGREGATION_TYPES.has(value);
 }
 
+const VALID_RUBRIC_OPERATORS: ReadonlySet<string> = new Set(RUBRIC_OPERATOR_VALUES);
+
+function parseRubricOperator(
+  value: unknown,
+  rubricId: string,
+  evaluatorName: string,
+  evalId: string,
+): RubricOperator | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+
+  if (typeof value === 'string' && VALID_RUBRIC_OPERATORS.has(value)) {
+    return value as RubricOperator;
+  }
+
+  logWarning(
+    `Ignoring invalid operator for rubric '${rubricId}' in evaluator '${evaluatorName}' in '${evalId}': must be one of ${RUBRIC_OPERATOR_VALUES.join(', ')}`,
+  );
+  return undefined;
+}
+
 /**
  * Parse rubric items from raw YAML/JSON data.
  * Supports both checklist rubrics and score-range rubrics.
@@ -1961,6 +1984,7 @@ function parseRubricItems(
 
     const id = asString(rawRubric.id) ?? `rubric-${index + 1}`;
     const expectedOutcome = asString(rawRubric.outcome) ?? '';
+    const operator = parseRubricOperator(rawRubric.operator, id, evaluatorName, evalId);
     const weight = typeof rawRubric.weight === 'number' ? rawRubric.weight : 1.0;
 
     // Parse min_score (0-1 scale), required_min_score (deprecated 0-10 scale), and required
@@ -2018,6 +2042,7 @@ function parseRubricItems(
         id,
         weight,
         ...(expectedOutcome.length > 0 ? { outcome: expectedOutcome } : {}),
+        ...(operator !== undefined ? { operator } : {}),
         ...(required !== undefined ? { required } : {}),
         ...(minScore !== undefined ? { min_score: minScore } : {}),
         ...(requiredMinScore !== undefined ? { required_min_score: requiredMinScore } : {}),
@@ -2035,6 +2060,7 @@ function parseRubricItems(
       items.push({
         id,
         outcome: expectedOutcome,
+        ...(operator !== undefined ? { operator } : {}),
         weight,
         // Default to required: true if not specified (backward compatibility)
         required: required ?? true,
@@ -2237,6 +2263,8 @@ export function parseInlineRubrics(
       }
 
       const expectedOutcome = asString(rubric.outcome) ?? '';
+      const id = asString(rubric.id) ?? `rubric-${index + 1}`;
+      const operator = parseRubricOperator(rubric.operator, id, 'rubrics', '<inline>');
 
       // Parse score_ranges if present (supports shorthand map format)
       const rawScoreRanges = rubric.score_ranges;
@@ -2256,7 +2284,8 @@ export function parseInlineRubrics(
           : undefined;
 
       const baseRubric = {
-        id: asString(rubric.id) ?? `rubric-${index + 1}`,
+        id,
+        ...(operator !== undefined ? { operator } : {}),
         weight: typeof rubric.weight === 'number' ? rubric.weight : 1.0,
       };
 

packages/core/src/evaluation/types.ts

@@ -467,6 +467,10 @@ export type ScoreRange = {
   readonly outcome: string;
 };
 
+export const RUBRIC_OPERATOR_VALUES = ['correctness', 'contradiction'] as const;
+
+export type RubricOperator = (typeof RUBRIC_OPERATOR_VALUES)[number];
+
 /**
  * Rubric item for LLM grader evaluation.
  * Supports two modes:
@@ -480,6 +484,11 @@ export type RubricItem = {
    * For score-range rubrics: optional overall criterion description.
    */
   readonly outcome?: string;
+  /**
+   * Optional grading intent. `correctness` requires positive support for the outcome.
+   * `contradiction` is a guard: omission is acceptable, but incompatible claims fail.
+   */
+  readonly operator?: RubricOperator;
   readonly weight: number;
   /**
    * Legacy boolean gating (treated as min_score: 1.0 for score-range rubrics).

packages/core/src/evaluation/validation/eval-file.schema.ts

@@ -72,6 +72,7 @@ const ScoreRangeSchema = z.object({
 const RubricItemSchema = z.object({
   id: z.string().optional(),
   outcome: z.string().optional(),
+  operator: z.enum(['correctness', 'contradiction']).optional(),
   weight: z.number().optional(),
   required: z.boolean().optional(),
   /** Minimum score (0-1) for this criterion to pass. */