diff --git a/apps/web/src/content/docs/docs/evaluation/rubrics.mdx b/apps/web/src/content/docs/docs/evaluation/rubrics.mdx index 14759e797..ba9502ea3 100644 --- a/apps/web/src/content/docs/docs/evaluation/rubrics.mdx +++ b/apps/web/src/content/docs/docs/evaluation/rubrics.mdx @@ -67,6 +67,7 @@ assertions: |-------|---------|-------------| | `id` | Auto-generated | Unique identifier for the criterion | | `outcome` | — | Description of what to check | +| `operator` | — | Optional intent hint: `correctness` or `contradiction` | | `weight` | `1.0` | Relative importance for scoring | | `required` | `false` | If true, failing this criterion fails the entire eval | | `min_score` | — | Minimum score (0–1) for this criterion to pass | @@ -76,6 +77,26 @@ assertions: `required_min_score` (0–10 integer scale) is deprecated. Use `min_score` (0–1 scale) instead. For example, `required_min_score: 8` becomes `min_score: 0.8`. ::: +### Criterion Operators + +Use `operator` when the criterion outcome should be interpreted with a specific grading intent instead of relying on the wording in `outcome`. + +```yaml +assertions: + - type: rubrics + criteria: + - id: supported-revenue + operator: correctness + outcome: States revenue increased to $10M + required: true + - id: no-revenue-conflict + operator: contradiction + outcome: Revenue increased to $10M + required: true +``` + +`correctness` requires the answer to positively satisfy the outcome. `contradiction` is a guard: the answer passes when it does not make an incompatible claim, even if it omits the outcome entirely. + ## Score-Range Mode (Analytic) For quality gradients instead of binary pass/fail, use score ranges: diff --git a/examples/features/rubric/README.md b/examples/features/rubric/README.md index bddcaa45a..8fdd3b788 100644 --- a/examples/features/rubric/README.md +++ b/examples/features/rubric/README.md @@ -7,6 +7,7 @@ Demonstrates rubric-based evaluation with weights, required flags, and auto-gene - Inline rubrics as strings - Rubric objects with weights - Required vs optional criteria +- Criterion operators for correctness and contradiction guards - Auto-generating rubrics from criteria - Rubric file references @@ -20,4 +21,5 @@ bun agentv eval examples/features/rubric/evals/dataset.eval.yaml --target defaul ## Key Files - `evals/dataset.eval.yaml` - Test cases with various rubric patterns +- `evals/operators.eval.yaml` - Focused example of correctness and contradiction operators - `evals/rubrics/` - External rubric files diff --git a/examples/features/rubric/evals/operators.eval.yaml b/examples/features/rubric/evals/operators.eval.yaml new file mode 100644 index 000000000..f0d116964 --- /dev/null +++ b/examples/features/rubric/evals/operators.eval.yaml @@ -0,0 +1,42 @@ +# AgentV Rubric Operator Example +# Demonstrates optional criterion operators for preserving rubric intent. + +name: rubric-operators +description: "Focused example showing correctness and contradiction rubric operators" + +execution: + target: llm + +tests: + - id: finance-summary-operator-guards + criteria: |- + Summarize the source faithfully. Include supported material facts and avoid + claims that contradict the source. + + input: + - role: user + content: |- + Source note: + - Q4 revenue increased to $10M from $8M. + - Gross margin declined to 42% from 45%. + + Write a two-sentence finance summary. + + expected_output: + - role: assistant + content: |- + Q4 revenue increased to $10M from $8M. Gross margin declined to 42% + from 45%. + + assertions: + - type: rubrics + criteria: + - id: supported-revenue + operator: correctness + outcome: States Q4 revenue increased to $10M from $8M. + required: true + + - id: no-margin-contradiction + operator: contradiction + outcome: Gross margin declined to 42% from 45%. + required: true diff --git a/packages/core/src/evaluation/graders/llm-grader-prompt.ts b/packages/core/src/evaluation/graders/llm-grader-prompt.ts index fc50aae4a..bb78dd39a 100644 --- a/packages/core/src/evaluation/graders/llm-grader-prompt.ts +++ b/packages/core/src/evaluation/graders/llm-grader-prompt.ts @@ -9,6 +9,7 @@ import { buildScoreRangeOutputSchema, substituteVariables, } from './llm-grader.js'; +import { formatRubricOperatorGuidance, formatRubricOperatorLabel } from './rubric-operators.js'; export interface LlmGraderPromptAssembly { systemPrompt: string; @@ -144,10 +145,20 @@ function assembleChecklist( parts.push('[[ ## rubrics ## ]]'); + const operatorGuidance = formatRubricOperatorGuidance(rubrics); + if (operatorGuidance.length > 0) { + parts.push('', 'Operator guidance:'); + for (const guidance of operatorGuidance) { + parts.push(`- ${guidance}`); + } + parts.push(''); + } + for (const rubric of rubrics) { const requiredLabel = rubric.required ? ' (REQUIRED)' : ''; const weightLabel = rubric.weight !== 1.0 ? ` (weight: ${rubric.weight})` : ''; - parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.outcome}`); + const operatorLabel = formatRubricOperatorLabel(rubric.operator); + parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}${operatorLabel}: ${rubric.outcome}`); } parts.push('', 'For each rubric, determine if it is satisfied and provide brief reasoning.'); @@ -213,6 +224,10 @@ function assembleScoreRange( parts.push('', `### Criterion: ${rubric.id}${weightLabel}${minScoreLabel}`); + if (rubric.operator) { + parts.push(`Operator: ${rubric.operator}`); + } + if (rubric.outcome) { parts.push(`Description: ${rubric.outcome}`); } @@ -227,6 +242,11 @@ function assembleScoreRange( } } + const operatorGuidance = formatRubricOperatorGuidance(rubrics); + if (operatorGuidance.length > 0) { + parts.push('', ...operatorGuidance); + } + parts.push( '', 'For each criterion, provide an integer score 0-10 that matches one of its defined score ranges.', diff --git a/packages/core/src/evaluation/graders/llm-grader.ts b/packages/core/src/evaluation/graders/llm-grader.ts index 985f574b8..3b6f58234 100644 --- a/packages/core/src/evaluation/graders/llm-grader.ts +++ b/packages/core/src/evaluation/graders/llm-grader.ts @@ -14,6 +14,7 @@ import { extractLastAssistantContent, isAgentProvider } from '../providers/types import { DEPRECATED_TEMPLATE_VARIABLES, TEMPLATE_VARIABLES } from '../template-variables.js'; import type { TokenUsage } from '../trace.js'; import type { AssertionEntry, JsonObject, RubricItem } from '../types.js'; +import { formatRubricOperatorGuidance, formatRubricOperatorLabel } from './rubric-operators.js'; import { clampScore, isNonEmptyString, parseJsonFromText, scoreToVerdict } from './scoring.js'; import type { EvaluationContext, EvaluationScore, Grader } from './types.js'; @@ -952,6 +953,10 @@ export class LlmGrader implements Grader { parts.push('', `### Criterion: ${rubric.id}${weightLabel}${minScoreLabel}`); + if (rubric.operator) { + parts.push(`Operator: ${rubric.operator}`); + } + if (rubric.outcome) { parts.push(`Description: ${rubric.outcome}`); } @@ -966,6 +971,11 @@ export class LlmGrader implements Grader { } } + const operatorGuidance = formatRubricOperatorGuidance(rubrics); + if (operatorGuidance.length > 0) { + parts.push('', ...operatorGuidance); + } + parts.push( '', 'For each criterion, provide an integer score 0-10 that matches one of its defined score ranges.', @@ -1007,10 +1017,22 @@ export class LlmGrader implements Grader { parts.push('[[ ## rubrics ## ]]'); + const operatorGuidance = formatRubricOperatorGuidance(rubrics); + if (operatorGuidance.length > 0) { + parts.push('', 'Operator guidance:'); + for (const guidance of operatorGuidance) { + parts.push(`- ${guidance}`); + } + parts.push(''); + } + for (const rubric of rubrics) { const requiredLabel = rubric.required ? ' (REQUIRED)' : ''; const weightLabel = rubric.weight !== 1.0 ? ` (weight: ${rubric.weight})` : ''; - parts.push(`- [${rubric.id}]${requiredLabel}${weightLabel}: ${rubric.outcome}`); + const operatorLabel = formatRubricOperatorLabel(rubric.operator); + parts.push( + `- [${rubric.id}]${requiredLabel}${weightLabel}${operatorLabel}: ${rubric.outcome}`, + ); } parts.push('', 'For each rubric, determine if it is satisfied and provide brief reasoning.'); diff --git a/packages/core/src/evaluation/graders/rubric-operators.ts b/packages/core/src/evaluation/graders/rubric-operators.ts new file mode 100644 index 000000000..fe4963d4e --- /dev/null +++ b/packages/core/src/evaluation/graders/rubric-operators.ts @@ -0,0 +1,27 @@ +import type { RubricItem, RubricOperator } from '../types.js'; + +const OPERATOR_GUIDANCE: Record = { + correctness: + 'Correctness: mark satisfied only when the answer positively supports or fulfills the outcome. Omission or contradiction should not satisfy it.', + contradiction: + 'Contradiction guard: mark satisfied when the answer does not make a claim that contradicts the outcome. Do not require the answer to mention the outcome; mark unsatisfied only for incompatible claims.', +}; + +export function formatRubricOperatorLabel(operator: RubricOperator | undefined): string { + return operator ? ` (operator: ${operator})` : ''; +} + +export function formatRubricOperatorGuidance(rubrics: readonly RubricItem[]): readonly string[] { + const operators = new Set(); + for (const rubric of rubrics) { + if (rubric.operator) { + operators.add(rubric.operator); + } + } + + if (operators.size === 0) { + return []; + } + + return [...operators].map((operator) => OPERATOR_GUIDANCE[operator]); +} diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index 7bdd9d01a..bde1b4cee 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -10,8 +10,9 @@ import type { GraderKind, JsonObject, JsonValue, + RubricOperator, } from '../types.js'; -import { isGraderKind } from '../types.js'; +import { RUBRIC_OPERATOR_VALUES, isGraderKind } from '../types.js'; import { validateCustomPromptContent } from '../validation/prompt-validator.js'; import { parseYamlValue } from '../yaml-loader.js'; import { resolveFileReference } from './file-resolver.js'; @@ -1940,6 +1941,28 @@ function isValidFieldAggregationType( return typeof value === 'string' && VALID_FIELD_AGGREGATION_TYPES.has(value); } +const VALID_RUBRIC_OPERATORS: ReadonlySet = new Set(RUBRIC_OPERATOR_VALUES); + +function parseRubricOperator( + value: unknown, + rubricId: string, + evaluatorName: string, + evalId: string, +): RubricOperator | undefined { + if (value === undefined) { + return undefined; + } + + if (typeof value === 'string' && VALID_RUBRIC_OPERATORS.has(value)) { + return value as RubricOperator; + } + + logWarning( + `Ignoring invalid operator for rubric '${rubricId}' in evaluator '${evaluatorName}' in '${evalId}': must be one of ${RUBRIC_OPERATOR_VALUES.join(', ')}`, + ); + return undefined; +} + /** * Parse rubric items from raw YAML/JSON data. * Supports both checklist rubrics and score-range rubrics. @@ -1961,6 +1984,7 @@ function parseRubricItems( const id = asString(rawRubric.id) ?? `rubric-${index + 1}`; const expectedOutcome = asString(rawRubric.outcome) ?? ''; + const operator = parseRubricOperator(rawRubric.operator, id, evaluatorName, evalId); const weight = typeof rawRubric.weight === 'number' ? rawRubric.weight : 1.0; // Parse min_score (0-1 scale), required_min_score (deprecated 0-10 scale), and required @@ -2018,6 +2042,7 @@ function parseRubricItems( id, weight, ...(expectedOutcome.length > 0 ? { outcome: expectedOutcome } : {}), + ...(operator !== undefined ? { operator } : {}), ...(required !== undefined ? { required } : {}), ...(minScore !== undefined ? { min_score: minScore } : {}), ...(requiredMinScore !== undefined ? { required_min_score: requiredMinScore } : {}), @@ -2035,6 +2060,7 @@ function parseRubricItems( items.push({ id, outcome: expectedOutcome, + ...(operator !== undefined ? { operator } : {}), weight, // Default to required: true if not specified (backward compatibility) required: required ?? true, @@ -2237,6 +2263,8 @@ export function parseInlineRubrics( } const expectedOutcome = asString(rubric.outcome) ?? ''; + const id = asString(rubric.id) ?? `rubric-${index + 1}`; + const operator = parseRubricOperator(rubric.operator, id, 'rubrics', ''); // Parse score_ranges if present (supports shorthand map format) const rawScoreRanges = rubric.score_ranges; @@ -2256,7 +2284,8 @@ export function parseInlineRubrics( : undefined; const baseRubric = { - id: asString(rubric.id) ?? `rubric-${index + 1}`, + id, + ...(operator !== undefined ? { operator } : {}), weight: typeof rubric.weight === 'number' ? rubric.weight : 1.0, }; diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 2027810a8..db9a01cb3 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -467,6 +467,10 @@ export type ScoreRange = { readonly outcome: string; }; +export const RUBRIC_OPERATOR_VALUES = ['correctness', 'contradiction'] as const; + +export type RubricOperator = (typeof RUBRIC_OPERATOR_VALUES)[number]; + /** * Rubric item for LLM grader evaluation. * Supports two modes: @@ -480,6 +484,11 @@ export type RubricItem = { * For score-range rubrics: optional overall criterion description. */ readonly outcome?: string; + /** + * Optional grading intent. `correctness` requires positive support for the outcome. + * `contradiction` is a guard: omission is acceptable, but incompatible claims fail. + */ + readonly operator?: RubricOperator; readonly weight: number; /** * Legacy boolean gating (treated as min_score: 1.0 for score-range rubrics). diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 81c2e59ca..a6a03a01f 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -72,6 +72,7 @@ const ScoreRangeSchema = z.object({ const RubricItemSchema = z.object({ id: z.string().optional(), outcome: z.string().optional(), + operator: z.enum(['correctness', 'contradiction']).optional(), weight: z.number().optional(), required: z.boolean().optional(), /** Minimum score (0-1) for this criterion to pass. */ diff --git a/packages/core/test/evaluation/graders.test.ts b/packages/core/test/evaluation/graders.test.ts index 2f4bb7be8..414dd001d 100644 --- a/packages/core/test/evaluation/graders.test.ts +++ b/packages/core/test/evaluation/graders.test.ts @@ -10,6 +10,7 @@ import { LlmGrader, TokenUsageGrader, } from '../../src/evaluation/graders.js'; +import { assembleLlmGraderPrompt } from '../../src/evaluation/graders/llm-grader-prompt.js'; import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; import type { Provider, @@ -516,6 +517,88 @@ describe('LlmGrader (llm-grader)', () => { ).toContain('[r2]'); }); + it('preserves correctness and contradiction operators in rubric prompts', async () => { + const graderProvider = new CapturingProvider({ + output: [ + { + role: 'assistant', + content: JSON.stringify({ + checks: [ + { id: 'supported-revenue', satisfied: true, reasoning: 'Supported by answer' }, + { id: 'no-revenue-conflict', satisfied: true, reasoning: 'No conflict present' }, + ], + overall_reasoning: 'Operators were preserved.', + }), + }, + ], + }); + + const evaluator = new LlmGrader({ + resolveGraderProvider: async () => graderProvider, + }); + + await evaluator.evaluate({ + evalCase: { ...baseTestCase, evaluator: 'llm-grader' }, + candidate: 'Revenue increased to $10M.', + target: baseTarget, + provider: graderProvider, + attempt: 0, + promptInputs: { question: '' }, + now: new Date(), + evaluator: { + name: 'rubric', + type: 'llm-grader', + rubrics: [ + { + id: 'supported-revenue', + operator: 'correctness', + outcome: 'States revenue increased to $10M', + weight: 1.0, + required: true, + }, + { + id: 'no-revenue-conflict', + operator: 'contradiction', + outcome: 'Revenue increased to $10M', + weight: 1.0, + required: true, + }, + ], + }, + }); + + const prompt = graderProvider.lastRequest?.question ?? ''; + expect(prompt).toContain('(operator: correctness)'); + expect(prompt).toContain('(operator: contradiction)'); + expect(prompt).toContain('Correctness: mark satisfied only when'); + expect(prompt).toContain('Contradiction guard: mark satisfied when'); + expect(prompt).toContain('Do not require the answer to mention the outcome'); + }); + + it('includes rubric operator guidance in shared prompt assembly', () => { + const prompt = assembleLlmGraderPrompt({ + evalCase: baseTestCase, + candidate: 'Revenue did not decline.', + promptInputs: { question: '' }, + evaluatorConfig: { + name: 'rubric', + type: 'llm-grader', + rubrics: [ + { + id: 'no-conflict', + operator: 'contradiction', + outcome: 'Revenue increased to $10M', + weight: 1.0, + required: true, + }, + ], + }, + }); + + expect(prompt.userPrompt).toContain('(operator: contradiction)'); + expect(prompt.userPrompt).toContain('Contradiction guard'); + }); + it('passes multi-turn role markers through to evaluator prompts', async () => { const graderProvider = new CapturingProvider({ output: [{ role: 'assistant', content: JSON.stringify({ score: 0.65, assertions: [] }) }], diff --git a/packages/core/test/evaluation/loaders/grader-parser.test.ts b/packages/core/test/evaluation/loaders/grader-parser.test.ts index de1984798..2a2ade206 100644 --- a/packages/core/test/evaluation/loaders/grader-parser.test.ts +++ b/packages/core/test/evaluation/loaders/grader-parser.test.ts @@ -1611,6 +1611,66 @@ describe('parseGraders - type: rubrics with criteria', () => { expect((evaluators?.[0] as LlmGraderConfig).weight).toBe(4.0); }); + it('preserves optional rubric criterion operators', async () => { + const evaluators = await parseGraders( + { + assertions: [ + { + type: 'rubrics', + criteria: [ + { + id: 'correct-fact', + operator: 'correctness', + outcome: 'Revenue increased to $10M', + }, + { + id: 'no-conflict', + operator: 'contradiction', + outcome: 'Revenue increased to $10M', + }, + ], + }, + ], + }, + undefined, + [tempDir], + 'test-1', + ); + + const config = evaluators?.[0] as LlmGraderConfig; + expect(config.rubrics?.[0]?.operator).toBe('correctness'); + expect(config.rubrics?.[1]?.operator).toBe('contradiction'); + }); + + it('ignores invalid rubric criterion operators without dropping the criterion', async () => { + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); + const evaluators = await parseGraders( + { + assertions: [ + { + type: 'rubrics', + criteria: [ + { + id: 'fact', + operator: 'unsupported', + outcome: 'Revenue increased to $10M', + }, + ], + }, + ], + }, + undefined, + [tempDir], + 'test-1', + ); + + const config = evaluators?.[0] as LlmGraderConfig; + expect(config.rubrics).toHaveLength(1); + expect(config.rubrics?.[0]?.operator).toBeUndefined(); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Ignoring invalid operator')); + warnSpy.mockRestore(); + }); + it('auto-generates name for rubrics type', async () => { const evaluators = await parseGraders( { diff --git a/packages/core/test/evaluation/rubric-operators-yaml.test.ts b/packages/core/test/evaluation/rubric-operators-yaml.test.ts new file mode 100644 index 000000000..44f36bbdf --- /dev/null +++ b/packages/core/test/evaluation/rubric-operators-yaml.test.ts @@ -0,0 +1,52 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { loadTests } from '../../src/evaluation/yaml-parser.js'; + +describe('rubric criterion operators', () => { + const tempDirs: string[] = []; + + afterEach(async () => { + await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true }))); + tempDirs.length = 0; + }); + + it('converts YAML operator fields into typed internal rubric items', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-rubric-operators-')); + tempDirs.push(dir); + + await writeFile( + path.join(dir, 'suite.eval.yaml'), + `tests: + - id: finance-summary + input: "Summarize the finance note" + criteria: "Keep supported facts and avoid contradictions" + assertions: + - type: rubrics + criteria: + - id: supported-revenue + operator: correctness + outcome: "States revenue increased to $10M" + - id: no-revenue-conflict + operator: contradiction + outcome: "Revenue increased to $10M" +`, + 'utf8', + ); + + const tests = await loadTests(path.join(dir, 'suite.eval.yaml'), dir); + const evaluator = tests[0]?.assertions?.[0]; + + expect(evaluator?.type).toBe('llm-grader'); + if (!evaluator || evaluator.type !== 'llm-grader') { + throw new Error('expected rubrics to normalize to llm-grader'); + } + + expect(evaluator.rubrics?.map((rubric) => rubric.operator)).toEqual([ + 'correctness', + 'contradiction', + ]); + }); +}); diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 9879d64e0..f2ac8ebc4 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -93,6 +93,32 @@ describe('validateEvalFile', () => { expect(result.errors).toHaveLength(0); }); + it('validates rubric criteria with optional operators', async () => { + const filePath = path.join(tempDir, 'rubric-operators.yaml'); + await writeFile( + filePath, + `tests: + - id: finance-summary + criteria: Keep supported facts and avoid contradictions + input: Summarize the finance note + assertions: + - type: rubrics + criteria: + - id: supported-revenue + operator: correctness + outcome: States revenue increased to $10M + - id: no-revenue-conflict + operator: contradiction + outcome: Revenue increased to $10M +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + it('rejects eval file without input field', async () => { const filePath = path.join(tempDir, 'missing-input.yaml'); await writeFile( diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 9291b3049..b2aa9c715 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -504,6 +504,7 @@ Binary check: is the output valid JSON? weight: 3.0 ``` LLM-judged structured evaluation with weighted criteria. Criteria items support `id`, `outcome`, `weight`, and `required` fields. +Use optional `operator: correctness` for positive support checks or `operator: contradiction` for guard criteria where omission is acceptable but incompatible claims fail. ### rubrics (inline, deprecated) Top-level `rubrics:` field is deprecated. Use `type: rubrics` under `assertions` instead. diff --git a/skills-data/agentv-eval-writer/references/eval-schema.json b/skills-data/agentv-eval-writer/references/eval-schema.json index 3a26739e6..fcaab0a0f 100644 --- a/skills-data/agentv-eval-writer/references/eval-schema.json +++ b/skills-data/agentv-eval-writer/references/eval-schema.json @@ -425,6 +425,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -1299,6 +1303,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -1564,6 +1572,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -2438,6 +2450,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -2924,6 +2940,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -3798,6 +3818,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -4063,6 +4087,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -4937,6 +4965,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -5664,6 +5696,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -6544,6 +6580,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -6959,6 +6999,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -7833,6 +7877,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -8098,6 +8146,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -8972,6 +9024,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -9458,6 +9514,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -10332,6 +10392,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -10597,6 +10661,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -11471,6 +11539,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -12198,6 +12270,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -13078,6 +13154,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -13597,6 +13677,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -14471,6 +14555,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -14736,6 +14824,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -15610,6 +15702,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -15926,6 +16022,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, @@ -16800,6 +16900,10 @@ "outcome": { "type": "string" }, + "operator": { + "type": "string", + "enum": ["correctness", "contradiction"] + }, "weight": { "type": "number" }, diff --git a/skills-data/agentv-eval-writer/references/rubric-evaluator.md b/skills-data/agentv-eval-writer/references/rubric-evaluator.md index 2520e9b60..141df7a77 100644 --- a/skills-data/agentv-eval-writer/references/rubric-evaluator.md +++ b/skills-data/agentv-eval-writer/references/rubric-evaluator.md @@ -16,6 +16,7 @@ Rubrics are defined as `assertions` entries with `type: rubrics`. They support b |-------|------|---------|-------------| | `id` | string | auto-generated | Unique identifier | | `outcome` | string | required* | Criterion being evaluated (*optional if `score_ranges` used) | +| `operator` | string | - | Optional intent: `correctness` or `contradiction` | | `weight` | number | 1.0 | Relative importance | | `required` | boolean | true | Failing forces verdict to 'fail' (checklist mode) | | `min_score` | number | - | Minimum score (0–1) to pass this criterion | @@ -62,6 +63,25 @@ assertions: required: false ``` +### Criterion Operators + +Use `operator` when outcome text should carry grading intent without embedding words like "must not contradict" in the outcome itself: + +```yaml +assertions: + - type: rubrics + criteria: + - id: supported-fact + operator: correctness + outcome: States revenue increased to $10M + - id: no-conflicting-fact + operator: contradiction + outcome: Revenue increased to $10M +``` + +- `correctness`: answer must positively support or fulfill the outcome. +- `contradiction`: answer may omit the outcome, but must not make an incompatible claim. + ## Score-Range Mode Shorthand map format (recommended):