Complete hard-deprecation codemod migrations (#1654)

* Complete hard-deprecation codemod migrations

* Fix manifest path codemod target

* Fix rerun env fixture syntax

Author: christso

apps/cli/src/templates/.agentv/targets.yaml

@@ -6,25 +6,25 @@
 targets:
   - id: default
     provider: azure
-    endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
-    api_key: ${{ AZURE_OPENAI_API_KEY }}
-    model: ${{ AZURE_DEPLOYMENT_NAME }}
-    # version: ${{ AZURE_OPENAI_API_VERSION }}  # Optional: uncomment to override default (v1)
+    endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
+    api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
+    model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
+    # version: "{{ env.AZURE_OPENAI_API_VERSION }}"  # Optional: uncomment to override default (v1)
 
   - id: codex
     provider: codex
     grader_target: azure-llm
     # Uses the Codex CLI (defaults to `codex` on PATH)
-    # executable: ${{ CODEX_CLI_PATH }}        # Optional: override executable path
+    # executable: "{{ env.CODEX_CLI_PATH }}"        # Optional: override executable path
     # args:                             # Optional additional CLI arguments
     #   - --profile
-    #   - ${{ CODEX_PROFILE }}
+    #   - "{{ env.CODEX_PROFILE }}"
     #   - --model
-    #   - ${{ CODEX_MODEL }}
+    #   - "{{ env.CODEX_MODEL }}"
     #   - --ask-for-approval
-    #   - ${{ CODEX_APPROVAL_PRESET }}
-    cwd: ${{ CODEX_WORKSPACE_DIR }}            # Where scratch workspaces are created
-    log_dir: ${{ CODEX_LOG_DIR }}              # Optional: where Codex CLI stream logs are stored (defaults to ./.agentv/logs/codex)
+    #   - "{{ env.CODEX_APPROVAL_PRESET }}"
+    cwd: "{{ env.CODEX_WORKSPACE_DIR }}"            # Where scratch workspaces are created
+    log_dir: "{{ env.CODEX_LOG_DIR }}"              # Optional: where Codex CLI stream logs are stored (defaults to ./.agentv/logs/codex)
     stream_log: raw                    # Optional: 'summary' for consolidated logs or 'raw' for per-event logs
 
   # Claude - Anthropic's Claude Agent SDK
@@ -33,24 +33,24 @@ targets:
     grader_target: azure-llm
     # Uses the @anthropic-ai/claude-agent-sdk
     # model: claude-sonnet-4-20250514          # Optional: override model
-    # cwd: ${{ CLAUDE_WORKSPACE_DIR }}         # Optional: working directory (defaults to process.cwd())
+    # cwd: "{{ env.CLAUDE_WORKSPACE_DIR }}"         # Optional: working directory (defaults to process.cwd())
     # max_turns: 50                            # Optional: max conversation turns
     # max_budget_usd: 5.0                      # Optional: max cost budget in USD
-    # log_dir: ${{ CLAUDE_LOG_DIR }}           # Optional: where stream logs are stored (defaults to ./.agentv/logs/claude)
+    # log_dir: "{{ env.CLAUDE_LOG_DIR }}"           # Optional: where stream logs are stored (defaults to ./.agentv/logs/claude)
     stream_log: raw                           # Optional: 'summary' for consolidated logs or 'raw' for per-event logs
     # system_prompt: optional override (default instructs agent to include code in response)
 
   - id: azure-llm
     provider: azure
-    endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
-    api_key: ${{ AZURE_OPENAI_API_KEY }}
-    model: ${{ AZURE_DEPLOYMENT_NAME }}
-    version: ${{ AZURE_OPENAI_API_VERSION }}
+    endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
+    api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
+    model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
+    version: "{{ env.AZURE_OPENAI_API_VERSION }}"
 
   - id: gemini-llm
     provider: gemini
-    api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }}
-    model: ${{ GEMINI_MODEL_NAME }}
+    api_key: "{{ env.GOOGLE_GENERATIVE_AI_API_KEY }}"
+    model: "{{ env.GEMINI_MODEL_NAME }}"
 
   - id: local_cli
     provider: cli
@@ -61,37 +61,37 @@ targets:
     # Format for each file in {FILES}. {path} and {basename} are automatically shell-escaped, so no quotes needed
     files_format: --file {path}
     # Optional working directory resolved from .env
-    cwd: ${{ CLI_EVALS_DIR }}
+    cwd: "{{ env.CLI_EVALS_DIR }}"
     healthcheck:
       command: uv run ./mock_cli.py --healthcheck
 
-  # ── MiMo (Xiaomi) via OpenRouter ───────────────────────────────────
+  # -- MiMo (Xiaomi) via OpenRouter -----------------------------------
   # All MiMo models are available through OpenRouter with OpenAI-compatible API.
   # See https://openrouter.ai/xiaomi/mimo-v2.5-pro for pricing and limits.
   #
   # Models:
-  #   mimo-v2.5-pro    — 1M context, 131K output, flagship
-  #   mimo-v2-pro      — 1M context, ~131K output
-  #   mimo-v2.5        — 1M context, ~131K output, multimodal
-  #   mimo-v2-flash    — 262K context, 65K output, fast MoE (open-source)
-  #   mimo-v2-omni     — 262K context, 65K output, omni-modal
+  #   mimo-v2.5-pro    - 1M context, 131K output, flagship
+  #   mimo-v2-pro      - 1M context, ~131K output
+  #   mimo-v2.5        - 1M context, ~131K output, multimodal
+  #   mimo-v2-flash    - 262K context, 65K output, fast MoE (open-source)
+  #   mimo-v2-omni     - 262K context, 65K output, omni-modal
   - id: mimo
     provider: openrouter
-    api_key: ${{ OPENROUTER_API_KEY }}
+    api_key: "{{ env.OPENROUTER_API_KEY }}"
     model: xiaomi/mimo-v2.5-pro
 
   - id: mimo-flash
     provider: openrouter
-    api_key: ${{ OPENROUTER_API_KEY }}
+    api_key: "{{ env.OPENROUTER_API_KEY }}"
     model: xiaomi/mimo-v2-flash
 
-  # ── Direct provider (not through OpenRouter) ───────────────────────
+  # -- Direct provider (not through OpenRouter) -----------------------
   # For providers not in pi-ai's model registry, set max_output_tokens
   # to match your model's actual output limit. Without this, the default
   # is 16K which may cap output below the model's capability.
   # - name: mimo-direct
   #   provider: openai
   #   base_url: https://token-plan-sgp.xiaomimimo.com/v1
-  #   api_key: ${{ XIAOMI_MIMO_API_KEY }}
+  #   api_key: "{{ env.XIAOMI_MIMO_API_KEY }}"
   #   model: xiaomi/mimo-v2.5-pro
   #   max_output_tokens: 131072

apps/cli/test/commands/runs/rerun.test.ts

@@ -280,7 +280,7 @@ describe('agentv runs rerun', () => {
     const created = await fixture(`targets:
   - id: captured
     provider: cli
-    command: \${{ LOCAL_AGENT_COMMAND }}
+    command: "{{ env.LOCAL_AGENT_COMMAND }}"
 `);
 
     const missing = await runCli(created, [
@@ -292,7 +292,7 @@ describe('agentv runs rerun', () => {
       '--dry-run',
     ]);
     expect(missing.exitCode).toBe(1);
-    expect(missing.stderr).toContain('Missing environment variable(s)');
+    expect(missing.stderr).toContain("Failed to resolve target 'captured'");
     expect(missing.stderr).toContain('LOCAL_AGENT_COMMAND');
 
     const withAmbientEnv = await runCli(

apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx

@@ -404,17 +404,29 @@ tests:
       - Assistant correctly explains the bug and proposes a fix
 ```
 
-Suite-level `preprocessors` apply to explicit LLM graders. That matters when the
-agent output is a `ContentFile` block rather than plain text:
+Suite-level transforms apply before graders see the target output. That matters
+when the agent output is a `ContentFile` block rather than plain text:
 
 ```yaml
-preprocessors:
-  - type: xlsx
-    command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"]
+default_test:
+  options:
+    transform: >-
+      return (() => {
+        const file = Array.isArray(output)
+          ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
+          : undefined;
+        if (!file) return output;
+        const result = Bun.spawnSync(["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"], {
+          stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
+        });
+        if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
+        return new TextDecoder().decode(result.stdout).trim();
+      })()
 
 tests:
   - id: spreadsheet-eval
-    input: Generate the spreadsheet report
+    vars:
+      input: Generate the spreadsheet report
     assert:
       - Output includes the revenue rows
 ```
@@ -466,22 +478,17 @@ tests:
         value: "fix"
 ```
 
-When you need a custom file conversion for only one grader, add `preprocessors` directly to that grader:
+When you need a custom output conversion for only one grader, add `transform` directly to that grader:
 
 ```yaml
-preprocessors:
-  - type: xlsx
-    command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"]
-
 tests:
   - id: mixed-eval
-    input: "Debug this function..."
+    vars:
+      input: "Debug this function..."
     assert:
       - Response is helpful and mentions the fix
-      - type: llm-rubric        # use explicit form for custom preprocessors
-        preprocessors:
-          - type: xlsx
-            command: ["bun", "run", "scripts/preprocessors/xlsx-to-json.ts"]
+      - type: llm-rubric
+        transform: file://scripts/transforms/xlsx-to-json.ts
       - type: contains
         value: "fix"
 ```

apps/web/src/content/docs/docs/next/evaluation/examples.mdx

@@ -101,22 +101,33 @@ tests:
       }
 ```
 
-## File Output Preprocessing
+## File Output Transform
 
 Convert a binary file output into text before the `llm-rubric` sees it:
 
 ```yaml
-description: Grade spreadsheet output via a preprocessor
-
-preprocessors:
-  - type: xlsx
-    command: ["bun", "run", "../scripts/preprocessors/xlsx-to-csv.ts"]
+description: Grade spreadsheet output via a transform
 
 target: file_output
+default_test:
+  options:
+    transform: >-
+      return (() => {
+        const file = Array.isArray(output)
+          ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
+          : undefined;
+        if (!file) return output;
+        const result = Bun.spawnSync(["bun", "run", "../scripts/preprocessors/xlsx-to-csv.ts"], {
+          stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
+        });
+        if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
+        return new TextDecoder().decode(result.stdout).trim();
+      })()
 
 tests:
   - id: spreadsheet-output
-    input: Generate the spreadsheet report
+    vars:
+      input: Generate the spreadsheet report
     criteria: The extracted spreadsheet content includes the revenue rows
     assert:
       - Output contains the transformed spreadsheet text including the revenue rows

apps/web/src/content/docs/docs/next/graders/llm-graders.mdx

@@ -192,20 +192,32 @@ assert:
 
 The `config` object is available as `ctx.config` inside the template function.
 
-## Preprocessing File Outputs
+## Transforming File Outputs
 
-If an agent returns a `ContentFile` block instead of plain text, you can preprocess that file into text before `llm-rubric` builds the candidate prompt.
+If an agent returns a `ContentFile` block instead of plain text, use `transform` to convert that file into text before `llm-rubric` builds the candidate prompt.
 
-AgentV always tries a default UTF-8 text read first. That is enough for text-based formats such as CSV, JSON, SQL, Markdown, YAML, HTML, XML, and plain text. For binary formats such as `.xlsx`, `.pdf`, or `.docx`, add a preprocessor command:
+AgentV always tries a default UTF-8 text read first. That is enough for text-based formats such as CSV, JSON, SQL, Markdown, YAML, HTML, XML, and plain text. For binary formats such as `.xlsx`, `.pdf`, or `.docx`, add a transform:
 
 ```yaml
-preprocessors:
-  - type: xlsx
-    command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"]
+default_test:
+  options:
+    transform: >-
+      return (() => {
+        const file = Array.isArray(output)
+          ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx"))
+          : undefined;
+        if (!file) return output;
+        const result = Bun.spawnSync(["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"], {
+          stdin: JSON.stringify({ path: file.path, media_type: file.media_type })
+        });
+        if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim());
+        return new TextDecoder().decode(result.stdout).trim();
+      })()
 
 tests:
   - id: spreadsheet-output
-    input: Generate the spreadsheet report
+    vars:
+      input: Generate the spreadsheet report
     assert:
       - Output includes the revenue rows
       - name: spreadsheet-check
@@ -216,15 +228,14 @@ tests:
           {{ output }}
 ```
 
-`type` accepts either a short alias such as `xlsx` or a full MIME type such as `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`.
-
 Resolution order:
 
-- per-grader `preprocessors` override suite-level entries
-- if no preprocessor matches, AgentV falls back to a UTF-8 text read
+- `tests[].options.transform` overrides `default_test.options.transform`
+- assertion-level `transform` applies to that grader only
+- if no transform is configured, AgentV falls back to a UTF-8 text read
 - if the fallback read looks binary or invalid, the grader receives a warning note instead of failing the test run
 
-See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable example with a file-producing target and a custom preprocessor script.
+See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable example with a file-producing target and a spreadsheet conversion script.
 
 ## Available Context Fields
 

apps/web/src/content/docs/docs/next/targets/cli-provider.mdx

@@ -25,9 +25,9 @@ graders:
   - id: azure-grader
     provider: azure
     config:
-      endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
-      api_key: ${{ AZURE_OPENAI_API_KEY }}
-      model: ${{ AZURE_DEPLOYMENT_NAME }}
+      endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
+      api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
+      model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
 
 defaults:
   target: my-agent
@@ -137,9 +137,9 @@ graders:
   - id: azure-grader
     provider: azure
     config:
-      endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
-      api_key: ${{ AZURE_OPENAI_API_KEY }}
-      model: ${{ AZURE_DEPLOYMENT_NAME }}
+      endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
+      api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
+      model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
 
 defaults:
   grader: azure-grader

apps/web/src/content/docs/docs/next/targets/coding-agents.mdx

@@ -90,7 +90,7 @@ targets:
           target: /results
           access: rw
       secrets:
-        OPENAI_API_KEY: ${{ OPENAI_API_KEY }}
+        OPENAI_API_KEY: "{{ env.OPENAI_API_KEY }}"
     config:
       command: ["codex", "exec", "--json"]
       model: gpt-5-codex
@@ -144,7 +144,7 @@ targets:
     runtime: host
     config:
       command: ["codex-eng", "exec", "--json"]
-      model: ${{ CODEX_MODEL }}
+      model: "{{ env.CODEX_MODEL }}"
 ```
 
 Use `codex-sdk` only when you intentionally want the Codex SDK path:
@@ -190,8 +190,8 @@ targets:
     config:
       command: ["pi"]
       subprovider: openrouter
-      model: ${{ OPENROUTER_MODEL }}
-      api_key: ${{ OPENROUTER_API_KEY }}
+      model: "{{ env.OPENROUTER_MODEL }}"
+      api_key: "{{ env.OPENROUTER_API_KEY }}"
 ```
 
 Use `pi-sdk` only when you intentionally want the SDK path:
@@ -266,8 +266,8 @@ targets:
     config:
       command: ["copilot"]
       subprovider: openai
-      base_url: ${{ OPENAI_ENDPOINT }}
-      api_key: ${{ OPENAI_API_KEY }}
+      base_url: "{{ env.OPENAI_ENDPOINT }}"
+      api_key: "{{ env.OPENAI_API_KEY }}"
       api_format: responses
 ```