docs(transform): publish promptfoo file transform example (#1661)

Author: christso

apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx

@@ -410,18 +410,7 @@ when the agent output is a `ContentFile` block rather than plain text:
 ```yaml
 default_test:
   options:
-    transform: >-
-      return (() => {
-        const file = Array.isArray(output)
-          ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
-          : undefined;
-        if (!file) return output;
-        const result = Bun.spawnSync(["bun", "run", "scripts/transforms/xlsx-to-csv.ts"], {
-          stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
-        });
-        if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
-        return new TextDecoder().decode(result.stdout).trim();
-      })()
+    transform: file://scripts/transforms/xlsx-to-csv.ts
 
 tests:
   - id: spreadsheet-eval
@@ -493,6 +482,12 @@ tests:
         value: "fix"
 ```
 
+`default_test.options.transform` is inherited by every test. A
+`tests[].options.transform` entry overrides that inherited transform for one
+test, and assertion-level `transform` applies only to that assertion after the
+test/default transform has produced the candidate text for the rest of the
+graders.
+
 ## Metadata
 
 Pass additional context through the `metadata` field:

apps/web/src/content/docs/docs/next/evaluation/examples.mdx

@@ -111,18 +111,7 @@ description: Grade spreadsheet output via a transform
 target: file_output
 default_test:
   options:
-    transform: >-
-      return (() => {
-        const file = Array.isArray(output)
-          ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
-          : undefined;
-        if (!file) return output;
-        const result = Bun.spawnSync(["bun", "run", "../scripts/transforms/xlsx-to-csv.ts"], {
-          stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
-        });
-        if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
-        return new TextDecoder().decode(result.stdout).trim();
-      })()
+    transform: file://../scripts/transforms/xlsx-to-csv.ts
 
 tests:
   - id: spreadsheet-output

apps/web/src/content/docs/docs/next/graders/llm-graders.mdx

@@ -201,18 +201,7 @@ AgentV always tries a default UTF-8 text read first. That is enough for text-bas
 ```yaml
 default_test:
   options:
-    transform: >-
-      return (() => {
-        const file = Array.isArray(output)
-          ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
-          : undefined;
-        if (!file) return output;
-        const result = Bun.spawnSync(["bun", "run", "scripts/transforms/xlsx-to-csv.ts"], {
-          stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
-        });
-        if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
-        return new TextDecoder().decode(result.stdout).trim();
-      })()
+    transform: file://scripts/transforms/xlsx-to-csv.ts
 
 tests:
   - id: spreadsheet-output
@@ -230,8 +219,9 @@ tests:
 
 Resolution order:
 
-- `tests[].options.transform` overrides `default_test.options.transform`
-- assertion-level `transform` applies to that grader only
+- `default_test.options.transform` applies to every test unless overridden
+- `tests[].options.transform` overrides `default_test.options.transform` for that test
+- assertion-level `transform` applies to that grader only, after the test/default transform
 - if no transform is configured, AgentV falls back to a UTF-8 text read
 - if the fallback read looks binary or invalid, the grader receives a warning note instead of failing the test run
 

apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx

@@ -24,6 +24,7 @@ implements equivalent semantics directly.
 | Align with Promptfoo | AgentV accepts the same concept, with `snake_case` where the field crosses the YAML boundary. |
 | Keep AgentV divergence | AgentV intentionally uses a different shape because it is clearer for repo-native agent evals. |
 | Keep AgentV extension | AgentV adds a capability that does not try to be Promptfoo-compatible. |
+| Removed/rejected surface | AgentV rejects this authored surface and points to the current supported field. |
 | Defer/future-scope | AgentV does not accept the Promptfoo surface yet. Use an AgentV primitive or wait for direct implementation. |
 
 ## Authored Config Matrix
@@ -34,11 +35,14 @@ implements equivalent semantics directly.
 | Test rows | `tests` can be inline rows or a case-file reference; rows carry `vars`, `assert`, metadata, prompt/provider filters, and expected data. | `tests` can be inline rows or a raw-case path; rows carry `vars`, `assert`, `expected_output`, metadata, workspace overrides, and run overrides. | Align with Promptfoo | AgentV also supports `imports.suites` and `imports.tests` for explicit composition. Raw cases do not own suite context. |
 | Variables | `tests[].vars` plus `defaultTest.vars`; prompt templates can reference top-level var names. | `tests[].vars` plus `default_test.vars`; templates can use `{{ name }}` or `{{ vars.name }}`. | Align with Promptfoo | Per-test vars override default vars by key. |
 | Default test | `defaultTest`, inline object or `file://` reference. | `default_test`, inline object or `file://` / `ref://` reference. | Align with Promptfoo | AgentV uses `snake_case` for YAML. Shared prompt matrix defaults belong in `default_test.vars`. |
+| Output transform | `defaultTest.options.transform`, `tests[].options.transform`, and assertion-level `transform`. | `default_test.options.transform`, `tests[].options.transform`, and assertion-level `transform`. | Align with Promptfoo | Use `transform` to shape provider output before grading, including file-output conversions such as `.xlsx` to text. `tests[].options.transform` overrides the inherited default transform; assertion-level `transform` is scoped to one grader. |
+| Deprecated `postprocess` | Older Promptfoo configs may still mention `postprocess`. | Rejected. | Removed/rejected surface | Use `transform`. AgentV intentionally does not adopt Promptfoo's deprecated `postprocess` spelling. |
 | Evaluate options | `evaluateOptions` for runtime controls. | `evaluate_options` for runtime controls. | Align with Promptfoo | AgentV uses `evaluate_options.repeat`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency`. |
 | Authored concurrency | Common Promptfoo usage includes runtime options such as `maxConcurrency`. | `evaluate_options.max_concurrency`. | Keep AgentV divergence | Do not author `execution.max_concurrency` or top-level `workers` in eval YAML. CLI `--workers` remains an operator override. |
 | Target selection | Promptfoo normal evals use `providers`; `targets` can alias providers in unified config. | Use top-level `target` for one system under test or top-level `targets` for a target matrix. | Keep AgentV divergence | AgentV reserves `provider` for the backend/adapter kind inside a target object. Top-level `providers` is rejected to avoid overloading that term. |
 | Target object identity | Provider options often use `id` for backend/provider spec and optional `label` for display or matching. | Target objects use stable `id` for target identity, `provider` for backend kind, optional `runtime`, and `config` for provider settings. | Keep AgentV divergence | AgentV does not copy Promptfoo's `label`/`id` baggage because `provider` already names the backend boundary. |
 | Direct authored input | Promptfoo prompt authoring normally goes through `prompts` plus vars. | Top-level `input` and inline `tests[].input` are removed from normal authored eval YAML. External raw-case imports may still carry internal input rows for compatibility. | Removed AgentV extension | Author prompt text, chat/system/user messages, and file-backed prompt content as `prompts`; put row data in `tests[].vars` and shared defaults in `default_test.vars`. |
+| Authored preprocessors | Not Promptfoo's canonical output-shaping surface. | Rejected in current authored YAML. | Removed/rejected surface | Use `transform` at `default_test.options`, `tests[].options`, or the assertion that needs the shaped output. Historical versioned docs may still show old preprocessor examples. |
 | Suite assertions | `assert` entries can be strings or typed assertion objects. | `assert` entries can be strings, typed assertion objects, script graders, or AgentV extension graders. | Align with Promptfoo | Plain strings become semantic rubric checks. Use `assert`, not `assertions`, in current authored eval YAML. |
 | Assertion grouping | `type: assert-set` with child `assert` entries, optional `config`, `metric`, `weight`, and `threshold`. | `type: assert-set` with child `assert`, optional `config`, metric names, weights, and parent threshold. | Align with Promptfoo | Parent `config` is inherited by child assertions; child `config` keys override shared parent keys. Without `threshold`, pass/fail follows nonzero-weight child assertions. With `threshold`, the weighted aggregate score determines pass/fail. `type: composite` is rejected; use `assert-set`. |
 | Deterministic assertion vocabulary | Common Promptfoo types include `contains`, `icontains`, `contains-any`, `contains-all`, `starts-with`, `regex`, `is-json`, `equals`, `latency`, `cost`, `javascript`, `python`, `webhook`, `similar`, and `llm-rubric`. | AgentV accepts the implemented overlap, including `contains`, `icontains`, `contains-any`, `contains-all`, `starts-with`, `regex`, `is-json`, `equals`, `latency`, `cost`, `javascript`, `python`, `webhook`, `similar`, and `llm-rubric`. | Align with Promptfoo | Unsupported Promptfoo assertion names error instead of silently becoming custom assertion names. |

examples/features/file-transforms/.agentv/providers/grader-check.ts

@@ -9,21 +9,23 @@ if (!promptFile || !outputFile) {
 
 const prompt = readFileSync(promptFile, 'utf8');
 const passed = prompt.includes('spreadsheet: revenue,total') && prompt.includes('Q1,42');
+const rubricIds = Array.from(prompt.matchAll(/- \[(rubric-\d+)\]/g), (match) => match[1]);
+const checkIds = rubricIds.length > 0 ? rubricIds : ['rubric-1'];
 
 writeFileSync(
   outputFile,
   JSON.stringify({
     text: JSON.stringify({
-      score: passed ? 1 : 0,
-      assertions: [
-        {
-          text: 'transformed file content reached the llm grader',
-          passed,
-          evidence: passed
-            ? 'found transformed spreadsheet text in prompt'
-            : 'transformed spreadsheet text missing from prompt',
-        },
-      ],
+      checks: checkIds.map((id) => ({
+        id,
+        satisfied: passed,
+        reasoning: passed
+          ? 'found transformed spreadsheet text in prompt'
+          : 'transformed spreadsheet text missing from prompt',
+      })),
+      overall_reasoning: passed
+        ? 'The transformed spreadsheet rows reached the rubric prompt.'
+        : 'The transformed spreadsheet rows did not reach the rubric prompt.',
     }),
   }),
   'utf8',

examples/features/file-transforms/.agentv/targets.yaml

@@ -1,12 +1,30 @@
 $schema: agentv-targets-v2.2
 targets:
   - id: file_output
-    provider: file-output
-    command: bun run .agentv/providers/file-output.ts {OUTPUT_FILE}
-    cwd: ..
+    provider: cli
+    runtime: host
+    config:
+      command: bun run .agentv/providers/file-output.ts {OUTPUT_FILE}
+      cwd: ..
     grader_target: grader_check
 
   - id: grader_check
-    provider: grader-check
-    command: bun run .agentv/providers/grader-check.ts {PROMPT_FILE} {OUTPUT_FILE}
-    cwd: ..
+    provider: cli
+    runtime: host
+    config:
+      command: bun run .agentv/providers/grader-check.ts {PROMPT_FILE} {OUTPUT_FILE}
+      cwd: ..
+
+  - id: file_output_live
+    provider: cli
+    runtime: host
+    config:
+      command: bun run .agentv/providers/file-output.ts {OUTPUT_FILE}
+      cwd: ..
+    grader_target: local-openai-grader
+
+  - id: local-openai-grader
+    provider: openai
+    base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
+    api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
+    model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"

examples/features/file-transforms/README.md

@@ -8,6 +8,7 @@ Demonstrates how `default_test.options.transform` turns `ContentFile` outputs in
 - an agent target returning a `ContentFile` block instead of plain text
 - an `llm-rubric` receiving transformed spreadsheet text
 - relative `ContentFile.path` resolution against the target workspace
+- a deterministic grader target for local validation and a live LLM grader target for dogfood
 
 ## Running
 
@@ -18,6 +19,15 @@ bun apps/cli/src/cli.ts eval examples/features/file-transforms/evals/suite.yaml
 
 Expected result: the eval passes because the grader sees the transformed spreadsheet text from `generated/report.xlsx`.
 
+To run the same file-output path with a live OpenAI-compatible grader:
+
+```bash
+LOCAL_OPENAI_PROXY_BASE_URL=http://127.0.0.1:10531/v1 \
+LOCAL_OPENAI_PROXY_API_KEY=dummy-local-key \
+LOCAL_OPENAI_PROXY_MODEL=gpt-5.4-mini \
+bun apps/cli/src/cli.ts eval examples/features/file-transforms/evals/suite.yaml --target file_output_live
+```
+
 ## Key Files
 
 - `evals/suite.yaml` - eval with `default_test.options.transform`

examples/features/file-transforms/evals/suite.yaml

@@ -10,22 +10,4 @@ tests:
       input: Generate the spreadsheet report
 default_test:
   options:
-    transform: >-
-      return (() => {
-        const content = Array.isArray(output) ? output : [];
-        const matchers = ["xlsx","application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",".xlsx"];
-        const file = content.find((block) => {
-          if (!block || block.type !== "file") return false;
-          const mediaType = typeof block.media_type === "string" ? block.media_type : "";
-          const filePath = typeof block.path === "string" ? block.path : "";
-          return matchers.some((matcher) => mediaType === matcher || filePath.endsWith(matcher));
-        });
-        if (!file || typeof file.path !== "string") return output;
-        const result = Bun.spawnSync(["bun","run","../scripts/transforms/xlsx-to-csv.ts"], {
-          stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
-        });
-        if (result.exitCode !== 0) {
-          throw new Error(new TextDecoder().decode(result.stderr).trim() || "transform command failed");
-        }
-        return new TextDecoder().decode(result.stdout).trim();
-      })()
+    transform: file://../scripts/transforms/xlsx-to-csv.ts

examples/features/file-transforms/scripts/transforms/xlsx-to-csv.ts

@@ -1,12 +1,69 @@
-import { readFileSync } from 'node:fs';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
 
-const payload = JSON.parse(readFileSync(0, 'utf8')) as { path?: string };
+type ContentFile = {
+  type?: string;
+  media_type?: string;
+  path?: string;
+};
 
-if (!payload.path) {
-  throw new Error('missing file path');
+const XLSX_MEDIA_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet';
+
+function findSpreadsheet(output: unknown): ContentFile | undefined {
+  if (!Array.isArray(output)) {
+    return undefined;
+  }
+
+  return output.find((block): block is ContentFile => {
+    if (!block || typeof block !== 'object') {
+      return false;
+    }
+    const file = block as ContentFile;
+    return (
+      file.type === 'file' &&
+      typeof file.path === 'string' &&
+      (file.path.endsWith('.xlsx') || file.media_type === XLSX_MEDIA_TYPE)
+    );
+  });
+}
+
+function spreadsheetPath(file: ContentFile): string {
+  if (!file.path) {
+    throw new Error('missing spreadsheet path');
+  }
+
+  if (path.isAbsolute(file.path)) {
+    return file.path;
+  }
+
+  return path.resolve(import.meta.dir, '..', '..', file.path);
 }
 
-// Example-only placeholder transformation. Copy this script into your project
-// and replace it with real spreadsheet extraction logic.
-console.log('spreadsheet: revenue,total');
-console.log('Q1,42');
+export default function transform(output: unknown): string {
+  const file = findSpreadsheet(output);
+  if (!file) {
+    throw new Error('expected a .xlsx ContentFile output');
+  }
+
+  const absolutePath = spreadsheetPath(file);
+  if (!existsSync(absolutePath)) {
+    throw new Error(`spreadsheet not found: ${file.path}`);
+  }
+
+  // Example-only placeholder conversion. Replace this with a real XLSX parser
+  // such as SheetJS or your project's existing spreadsheet extractor.
+  return ['spreadsheet: revenue,total', 'Q1,42'].join('\n');
+}
+
+if (import.meta.main) {
+  const filePath = process.argv[2];
+  if (!filePath) {
+    throw new Error('missing file path');
+  }
+  const payload: ContentFile = {
+    type: 'file',
+    media_type: XLSX_MEDIA_TYPE,
+    path: filePath,
+  };
+  process.stdout.write(`${transform([payload])}\n`);
+}

skills-data/agentv-eval-migrations/references/breaking-changes.md

@@ -48,9 +48,12 @@ For a v4.42.4-era eval:
 13. Replace workspace hook `script:` with `command:`.
 14. For executable setup, prefer top-level `extensions`; keep
     `workspace.hooks.after_each.reset` for reset policy.
-15. Keep raw cases under `tests` or `imports.tests`; import full eval suites
+15. Replace authored `preprocessors` and deprecated Promptfoo `postprocess`
+    with `transform` at `default_test.options`, `tests[].options`, or the
+    assertion that needs the shaped output.
+16. Keep raw cases under `tests` or `imports.tests`; import full eval suites
     with `imports.suites`.
-16. Validate with `bun apps/cli/src/cli.ts validate <eval-file>`.
+17. Validate with `bun apps/cli/src/cli.ts validate <eval-file>`.
 
 ## Assertions Renamed To `assert`