fix(eval): remove public preprocessor authoring (#1656)

christso · web-flow · commit 94e91a5b86f4 · 2026-07-05T13:26:31.000+10:00
diff --git a/README.md b/README.md
@@ -113,7 +113,7 @@ Plain assertion strings are short-form rubric criteria: AgentV groups them into
 `llm-rubric` and writes each criterion to `grading.json.assertion_results` for the
 Dashboard. Use explicit `type: llm-rubric` when you need weights, required flags, or
 `score_ranges`, or when you need a custom grader prompt, grader target, or
-preprocessing; use string `value` for free-form rubric checks. Executable
+output transforms; use string `value` for free-form rubric checks. Executable
 graders use `type: script`.
 
 The target can be an eval-local object when this eval needs target settings of its own:
diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx
@@ -416,7 +416,7 @@ default_test:
           ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
           : undefined;
         if (!file) return output;
-        const result = Bun.spawnSync(["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"], {
+        const result = Bun.spawnSync(["bun", "run", "scripts/transforms/xlsx-to-csv.ts"], {
           stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
         });
         if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
diff --git a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx
@@ -117,7 +117,7 @@ default_test:
           ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
           : undefined;
         if (!file) return output;
-        const result = Bun.spawnSync(["bun", "run", "../scripts/preprocessors/xlsx-to-csv.ts"], {
+        const result = Bun.spawnSync(["bun", "run", "../scripts/transforms/xlsx-to-csv.ts"], {
           stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
         });
         if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
@@ -133,7 +133,7 @@ tests:
       - Output contains the transformed spreadsheet text including the revenue rows
 ```
 
-See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable end-to-end example with a file-producing target and custom grader target.
+See [`examples/features/file-transforms/`](../../../../../examples/features/file-transforms/) for a runnable end-to-end example with a file-producing target and custom grader target.
 
 ## Tool Trajectory
 
diff --git a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx
@@ -12,7 +12,7 @@ LLM graders use a language model to evaluate agent responses against custom crit
 
 Put semantic grading requirements in `assert`. Plain strings are
 handled by the built-in `llm-rubric` rubric grader. Use `type: llm-rubric` when you
-need a custom prompt, target, or grader-specific preprocessing:
+need a custom prompt, target, or grader-specific transform:
 
 ```yaml
 tests:
@@ -207,7 +207,7 @@ default_test:
           ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
           : undefined;
         if (!file) return output;
-        const result = Bun.spawnSync(["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"], {
+        const result = Bun.spawnSync(["bun", "run", "scripts/transforms/xlsx-to-csv.ts"], {
           stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
         });
         if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
@@ -235,7 +235,7 @@ Resolution order:
 - if no transform is configured, AgentV falls back to a UTF-8 text read
 - if the fallback read looks binary or invalid, the grader receives a warning note instead of failing the test run
 
-See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable example with a file-producing target and a spreadsheet conversion script.
+See [`examples/features/file-transforms/`](../../../../../examples/features/file-transforms/) for a runnable example with a file-producing target and a spreadsheet conversion script.
 
 ## Available Context Fields
 
diff --git a/examples/features/README.md b/examples/features/README.md
@@ -22,7 +22,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 | [assert-set](assert-set/) | Safety gate and weighted assertion groups |
 | [threshold-grader](threshold-grader/) | Pass a test if a configurable percentage of sub-graders pass |
 | [multi-turn-conversation](multi-turn-conversation/) | Grade a multi-turn conversation with per-turn score breakdowns |
-| [preprocessors](preprocessors/) | Convert `ContentFile` outputs with `default_test.options.transform` before `llm-rubric` runs |
+| [file-transforms](file-transforms/) | Convert `ContentFile` outputs with `default_test.options.transform` before `llm-rubric` runs |
 
 ---
 
@@ -159,7 +159,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 | [local-cli](local-cli/) | Workspace & targets |
 | [multi-turn-conversation](multi-turn-conversation/) | LLM grading |
 | [nlp-metrics](nlp-metrics/) | Deterministic assertions |
-| [preprocessors](preprocessors/) | LLM grading |
+| [file-transforms](file-transforms/) | LLM grading |
 | [prompt-template-sdk](prompt-template-sdk/) | TypeScript SDK |
 | [repo-lifecycle](repo-lifecycle/) | Workspace & targets |
 | [rubric](rubric/) | LLM grading |
diff --git a/examples/features/file-transforms/.agentv/providers/file-output.ts b/examples/features/file-transforms/.agentv/providers/file-output.ts
diff --git a/examples/features/file-transforms/.agentv/providers/grader-check.ts b/examples/features/file-transforms/.agentv/providers/grader-check.ts
@@ -17,7 +17,7 @@ writeFileSync(
       score: passed ? 1 : 0,
       assertions: [
         {
-          text: 'preprocessed file content reached the llm grader',
+          text: 'transformed file content reached the llm grader',
           passed,
           evidence: passed
             ? 'found transformed spreadsheet text in prompt'
diff --git a/examples/features/file-transforms/.agentv/targets.yaml b/examples/features/file-transforms/.agentv/targets.yaml
diff --git a/examples/features/file-transforms/README.md b/examples/features/file-transforms/README.md
@@ -13,7 +13,7 @@ Demonstrates how `default_test.options.transform` turns `ContentFile` outputs in
 
 ```bash
 # From repository root
-bun apps/cli/src/cli.ts eval examples/features/preprocessors/evals/suite.yaml --target file_output
+bun apps/cli/src/cli.ts eval examples/features/file-transforms/evals/suite.yaml --target file_output
 ```
 
 Expected result: the eval passes because the grader sees the transformed spreadsheet text from `generated/report.xlsx`.
@@ -24,4 +24,4 @@ Expected result: the eval passes because the grader sees the transformed spreads
 - `.agentv/targets.yaml` - custom file-producing target and custom grader target
 - `.agentv/providers/file-output.ts` - emits a relative `ContentFile` path
 - `.agentv/providers/grader-check.ts` - passes only when transformed text reaches the grader prompt
-- `scripts/preprocessors/xlsx-to-csv.ts` - example spreadsheet conversion script
+- `scripts/transforms/xlsx-to-csv.ts` - example spreadsheet conversion script
diff --git a/examples/features/file-transforms/evals/suite.yaml b/examples/features/file-transforms/evals/suite.yaml
@@ -21,11 +21,11 @@ default_test:
           return matchers.some((matcher) => mediaType === matcher || filePath.endsWith(matcher));
         });
         if (!file || typeof file.path !== "string") return output;
-        const result = Bun.spawnSync(["bun","run","../scripts/preprocessors/xlsx-to-csv.ts"], {
+        const result = Bun.spawnSync(["bun","run","../scripts/transforms/xlsx-to-csv.ts"], {
           stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
         });
         if (result.exitCode !== 0) {
-          throw new Error(new TextDecoder().decode(result.stderr).trim() || "preprocessor command failed");
+          throw new Error(new TextDecoder().decode(result.stderr).trim() || "transform command failed");
         }
         return new TextDecoder().decode(result.stdout).trim();
       })()
diff --git a/examples/features/file-transforms/scripts/transforms/xlsx-to-csv.ts b/examples/features/file-transforms/scripts/transforms/xlsx-to-csv.ts
diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts
@@ -569,13 +569,7 @@ async function parseGraderList(
     const pushEvaluator = (config: GraderConfig): void => {
       evaluators.push(transform !== undefined ? { ...config, transform } : config);
     };
-    const mergedPreprocessors = await parseMergedPreprocessors(
-      rawEvaluator.preprocessors as JsonValue | undefined,
-      defaultPreprocessors,
-      searchRoots,
-      name,
-      evalId,
-    );
+    const inheritedInternalPreprocessors = defaultPreprocessors;
 
     // Custom assertion types — store with their type name for registry dispatch
     if (isCustomType) {
@@ -788,7 +782,9 @@ async function parseGraderList(
         ...(min_score !== undefined ? { min_score } : {}),
         ...(negate !== undefined ? { negate } : {}),
         ...(Object.keys(mergedConfig).length > 0 ? { config: mergedConfig } : {}),
-        ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}),
+        ...(inheritedInternalPreprocessors
+          ? { preprocessors: inheritedInternalPreprocessors }
+          : {}),
         ...(targetConfig !== undefined ? { target: targetConfig } : {}),
       });
       continue;
@@ -1684,7 +1680,9 @@ async function parseGraderList(
         ...(finalConfig ? { config: finalConfig } : {}),
         ...(llmMaxSteps !== undefined ? { max_steps: llmMaxSteps } : {}),
         ...(llmTemperature !== undefined ? { temperature: llmTemperature } : {}),
-        ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}),
+        ...(inheritedInternalPreprocessors
+          ? { preprocessors: inheritedInternalPreprocessors }
+          : {}),
       });
       continue;
     }
@@ -1705,7 +1703,7 @@ async function parseGraderList(
       ...(finalConfig ? { config: finalConfig } : {}),
       ...(llmMaxSteps !== undefined ? { max_steps: llmMaxSteps } : {}),
       ...(llmTemperature !== undefined ? { temperature: llmTemperature } : {}),
-      ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}),
+      ...(inheritedInternalPreprocessors ? { preprocessors: inheritedInternalPreprocessors } : {}),
     });
   }
 
diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts
@@ -963,7 +963,7 @@ export interface EvalSourceReference {
     | 'code_grader_cwd'
     | 'assertion_template'
     | 'default_test'
-    | 'preprocessor_command';
+    | 'content_transform_command';
   readonly displayPath: string;
   readonly resolvedPath?: string;
   readonly graderName?: string;
@@ -1063,7 +1063,7 @@ export interface EvalTest {
   readonly vars?: JsonObject;
   /** Promptfoo-compatible output transform inherited from default_test.options or tests[].options. */
   readonly outputTransform?: TransformSpec;
-  /** Suite-level preprocessors used by the implicit default llm-grader. */
+  /** Internal content conversion commands for file blocks. Authored eval YAML uses transform. */
   readonly preprocessors?: readonly ContentPreprocessorConfig[];
   /** Promptfoo-style lifecycle extensions inherited from the suite. */
   readonly extensions?: readonly AgentVExtensionConfig[];
diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts
@@ -49,7 +49,6 @@ import {
   coerceEvaluator,
   collectAssertionTemplateSourceReferences,
   parseGraders,
-  parsePreprocessors,
   warnUnconsumedCriteria,
 } from './loaders/grader-parser.js';
 import { detectFormat, loadTestsFromJsonl } from './loaders/jsonl-parser.js';
@@ -1278,12 +1277,6 @@ async function loadTestsFromParsedYamlValue(
   const evalFileDir = path.dirname(absoluteTestPath);
 
   const globalEvaluator = coerceEvaluator(suite.evaluator, 'global');
-  const suitePreprocessors = await parsePreprocessors(
-    suite.preprocessors,
-    searchRoots,
-    '<suite>',
-    absoluteTestPath,
-  );
   const defaultTestRubricPrompt = extractDefaultTestRubricPrompt(suite);
   const suiteExtensions = parseExtensions(suite.extensions, evalFileDir);
 
@@ -1554,7 +1547,7 @@ async function loadTestsFromParsedYamlValue(
           globalExecution,
           searchRoots,
           id ?? 'unknown',
-          suitePreprocessors,
+          undefined,
           defaultTestRubricPrompt,
         );
       } catch (error) {
@@ -1650,7 +1643,6 @@ async function loadTestsFromParsedYamlValue(
         assertions: evaluators,
         ...(caseVars ? { vars: caseVars } : {}),
         ...(outputTransform ? { outputTransform } : {}),
-        ...(suitePreprocessors ? { preprocessors: suitePreprocessors } : {}),
         ...(suiteExtensions.length > 0 ? { extensions: suiteExtensions } : {}),
         workspace: mergedWorkspace,
         metadata,
@@ -2694,7 +2686,7 @@ function collectSingleGraderSourceReferences(
   for (const preprocessor of preprocessors ?? []) {
     if (preprocessor.resolvedCommand && preprocessor.resolvedCommand.length > 0) {
       references.push({
-        kind: 'preprocessor_command',
+        kind: 'content_transform_command',
         displayPath: preprocessor.resolvedCommand.at(-1) ?? preprocessor.type,
         resolvedPath: preprocessor.resolvedCommand.at(-1),
         graderName: evaluator.name,
diff --git a/packages/sdk/README.md b/packages/sdk/README.md
@@ -199,7 +199,7 @@ export default defineEval({
 });
 ```
 
-The helpers return ordinary `assert` entries such as `type: contains`, `type: llm-rubric`, and `type: script`. CamelCase SDK options such as `minScore` and `maxSteps` lower to canonical YAML keys such as `min_score` and `max_steps`.
+The helpers return ordinary `assert` entries such as `type: contains`, `type: llm-rubric`, and `type: script`. Use the shared `transform` option for assertion-level output shaping. CamelCase SDK options such as `minScore` and `maxSteps` lower to canonical YAML keys such as `min_score` and `max_steps`.
 
 If you are coming from Braintrust `scores` or DeepEval metrics, model reusable checks as small AgentV-native helper factories that return these grader configs. They still lower to the same YAML/runtime contract:
 
diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts
@@ -75,12 +75,6 @@ export interface EvalAssertionConfig {
   readonly [key: string]: unknown;
 }
 
-export interface EvalPreprocessor {
-  readonly type: string;
-  readonly command: string | readonly string[];
-  readonly [key: string]: unknown;
-}
-
 export interface EvalWorkspaceHook {
   readonly command?: string | readonly string[];
   readonly timeoutMs?: number;
@@ -229,7 +223,6 @@ export interface EvalDefinition {
   readonly threshold?: number;
   readonly budgetUsd?: number;
   readonly assert?: readonly EvalAssertionConfig[];
-  readonly preprocessors?: readonly EvalPreprocessor[];
   readonly workspace?: EvalWorkspace | string;
 }
 
@@ -314,6 +307,11 @@ function validateTopLevelRuntimeFields(definition: EvalDefinition): void {
       "defineEval() does not accept top-level 'input'. Use prompts with default test vars or tests[].vars instead.",
     );
   }
+  if (Object.prototype.hasOwnProperty.call(rawDefinition, 'preprocessors')) {
+    throw new Error(
+      "defineEval() does not accept top-level 'preprocessors'. Use defaultTest.options.transform or assertion-level transform instead.",
+    );
+  }
   if (
     Object.prototype.hasOwnProperty.call(rawDefinition, 'experiment') &&
     typeof rawDefinition.experiment !== 'string'
diff --git a/packages/sdk/src/graders.ts b/packages/sdk/src/graders.ts
@@ -1,4 +1,4 @@
-import type { EvalAssertionConfig, EvalPreprocessor } from './eval.js';
+import type { EvalAssertionConfig } from './eval.js';
 
 export type GraderCommand = string | readonly string[];
 
@@ -8,6 +8,7 @@ export interface GraderHelperOptions {
   readonly required?: boolean;
   readonly minScore?: number;
   readonly negate?: boolean;
+  readonly transform?: string;
 }
 
 export interface GraderCommonConfig {
@@ -16,6 +17,7 @@ export interface GraderCommonConfig {
   readonly required?: boolean;
   readonly minScore?: number;
   readonly negate?: boolean;
+  readonly transform?: string;
 }
 
 export interface ContainsGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
@@ -70,7 +72,6 @@ export interface LlmRubricGraderConfig extends EvalAssertionConfig, GraderCommon
   readonly config?: Readonly<Record<string, unknown>>;
   readonly maxSteps?: number;
   readonly temperature?: number;
-  readonly preprocessors?: readonly EvalPreprocessor[];
 }
 
 export interface GraderPromptScriptConfig {
@@ -85,7 +86,6 @@ export interface LlmGraderOptions extends GraderHelperOptions {
   readonly config?: Readonly<Record<string, unknown>>;
   readonly maxSteps?: number;
   readonly temperature?: number;
-  readonly preprocessors?: readonly EvalPreprocessor[];
 }
 
 export interface LlmGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
@@ -96,7 +96,6 @@ export interface LlmGraderConfig extends EvalAssertionConfig, GraderCommonConfig
   readonly config?: Readonly<Record<string, unknown>>;
   readonly maxSteps?: number;
   readonly temperature?: number;
-  readonly preprocessors?: readonly EvalPreprocessor[];
 }
 
 export interface ScriptGraderTargetOptions {
@@ -107,7 +106,6 @@ export interface ScriptGraderOptions extends GraderHelperOptions {
   readonly cwd?: string;
   readonly target?: true | ScriptGraderTargetOptions;
   readonly config?: Readonly<Record<string, unknown>>;
-  readonly preprocessors?: readonly EvalPreprocessor[];
 }
 
 export interface ScriptGraderConfig extends EvalAssertionConfig, GraderCommonConfig {
@@ -116,7 +114,6 @@ export interface ScriptGraderConfig extends EvalAssertionConfig, GraderCommonCon
   readonly cwd?: string;
   readonly target?: true | ScriptGraderTargetOptions;
   readonly config?: Readonly<Record<string, unknown>>;
-  readonly preprocessors?: readonly EvalPreprocessor[];
 }
 
 /** @deprecated Use ScriptGraderTargetOptions. */
@@ -148,6 +145,7 @@ function withCommon<T extends { readonly type: string }>(
     ...(options.required !== undefined ? { required: options.required } : {}),
     ...(options.minScore !== undefined ? { minScore: options.minScore } : {}),
     ...(options.negate !== undefined ? { negate: options.negate } : {}),
+    ...(options.transform !== undefined ? { transform: options.transform } : {}),
   } as T & GraderCommonConfig & EvalAssertionConfig;
 }
 
@@ -196,7 +194,6 @@ export function llmRubricGrader(
     readonly config?: Readonly<Record<string, unknown>>;
     readonly maxSteps?: number;
     readonly temperature?: number;
-    readonly preprocessors?: readonly EvalPreprocessor[];
   } = {},
 ): LlmRubricGraderConfig {
   return withCommon(
@@ -208,7 +205,6 @@ export function llmRubricGrader(
       ...(options.config !== undefined ? { config: options.config } : {}),
       ...(options.maxSteps !== undefined ? { maxSteps: options.maxSteps } : {}),
       ...(options.temperature !== undefined ? { temperature: options.temperature } : {}),
-      ...(options.preprocessors !== undefined ? { preprocessors: options.preprocessors } : {}),
     },
     options,
   );
@@ -224,7 +220,6 @@ export function llmGrader(options: LlmGraderOptions = {}): LlmGraderConfig {
       ...(options.config !== undefined ? { config: options.config } : {}),
       ...(options.maxSteps !== undefined ? { maxSteps: options.maxSteps } : {}),
       ...(options.temperature !== undefined ? { temperature: options.temperature } : {}),
-      ...(options.preprocessors !== undefined ? { preprocessors: options.preprocessors } : {}),
     },
     options,
   );
@@ -249,7 +244,6 @@ export function scriptGrader(
       ...(options.cwd !== undefined ? { cwd: options.cwd } : {}),
       ...(options.target !== undefined ? { target: options.target } : {}),
       ...(options.config !== undefined ? { config: options.config } : {}),
-      ...(options.preprocessors !== undefined ? { preprocessors: options.preprocessors } : {}),
     },
     options,
   );
diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts
@@ -145,7 +145,6 @@ export {
   type EvalExecution,
   type EvalMessage,
   type EvalMessageContent,
-  type EvalPreprocessor,
   type EvalRequires,
   type EvalTargetRef,
   type EvalTest,
diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts
@@ -261,6 +261,17 @@ describe('YAML-aligned eval authoring helpers', () => {
     ).toThrow(/tests\[0\]\.input/);
   });
 
+  it('rejects removed public preprocessor authoring', () => {
+    expect(() =>
+      defineEval({
+        name: 'removed-preprocessors',
+        prompts: ['{{ input }}'],
+        preprocessors: [{ type: 'xlsx', command: ['bun', 'run', 'xlsx-to-text.ts'] }],
+        tests: [{ id: 'hello', vars: { input: 'Say hello' } }],
+      } as never),
+    ).toThrow(/top-level 'preprocessors'.*defaultTest\.options\.transform/);
+  });
+
   it('rejects removed experiment authoring blocks', () => {
     expect(() =>
       defineEval({
diff --git a/packages/sdk/test/grader-helpers.test.ts b/packages/sdk/test/grader-helpers.test.ts
diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ writeFileSync(`
`17`	`17`	`score: passed ? 1 : 0,`
`18`	`18`	`assertions: [`
`19`	`19`	`{`
`20`		`- text: 'preprocessed file content reached the llm grader',`
	`20`	`+ text: 'transformed file content reached the llm grader',`
`21`	`21`	`passed,`
`22`	`22`	`evidence: passed`
`23`	`23`	`? 'found transformed spreadsheet text in prompt'`