fix(cache): honor configured response cache paths

Bead: av-vwa.3

Author: christso

apps/cli/src/commands/eval/commands/run.ts

@@ -108,9 +108,14 @@ export const evalRunCommand = command({
       long: 'cache',
       description: 'Enable provider response cache (persisted to disk)',
     }),
+    cachePath: option({
+      type: optional(string),
+      long: 'cache-path',
+      description: 'Enable provider response cache at the given directory',
+    }),
     noCache: flag({
       long: 'no-cache',
-      description: 'Disable caching (overrides YAML execution.cache)',
+      description: 'Disable response caching (overrides --cache, --cache-path, config, and YAML)',
     }),
     verbose: flag({
       long: 'verbose',
@@ -262,6 +267,7 @@ export const evalRunCommand = command({
       agentTimeout: args.agentTimeout,
       maxRetries: args.maxRetries,
       cache: args.cache,
+      cachePath: args.cachePath,
       noCache: args.noCache,
       verbose: args.verbose,
       workspaceMode: args.workspaceMode,

apps/cli/src/commands/eval/run-eval.ts

@@ -96,7 +96,10 @@ interface NormalizedOptions {
   readonly agentTimeoutSeconds?: number;
   readonly maxRetries: number;
   readonly cache: boolean;
+  readonly cachePath?: string;
   readonly noCache: boolean;
+  readonly tsConfigCache?: boolean;
+  readonly tsConfigCachePath?: string;
   readonly verbose: boolean;
   readonly otelFile?: string;
   readonly exportOtel: boolean;
@@ -313,13 +316,12 @@ function normalizeOptions(
   const cliMaxRetries = normalizeOptionalNumber(rawOptions.maxRetries);
   const configMaxRetries = config?.execution?.maxRetries;
 
-  // Cache: CLI flags take priority, then config file, then default (true via shouldEnableCache)
-  const cliCache = normalizeBoolean(rawOptions.cache);
+  // Response cache: CLI request/path, then eval YAML, then TypeScript config, then default off.
+  const cliCachePath = normalizeString(rawOptions.cachePath);
+  const cliCache = normalizeBoolean(rawOptions.cache) || cliCachePath !== undefined;
   const cliNoCache = normalizeBoolean(rawOptions.noCache);
   const configCacheEnabled = config?.cache?.enabled;
-  // If neither --cache nor --no-cache was passed, use config value
-  const resolvedCache = cliCache || (!cliNoCache && configCacheEnabled === true);
-  const resolvedNoCache = cliNoCache;
+  const configCachePath = normalizeString(config?.cache?.path);
 
   // Output dir: CLI --out > config output.dir > auto-generated
   const cliOut = normalizeString(rawOptions.out);
@@ -352,8 +354,11 @@ function normalizeOptions(
     dryRunDelayMax: normalizeNumber(rawOptions.dryRunDelayMax, 0),
     agentTimeoutSeconds: cliAgentTimeout ?? configAgentTimeoutSeconds,
     maxRetries: cliMaxRetries ?? configMaxRetries ?? 2,
-    cache: resolvedCache,
-    noCache: resolvedNoCache,
+    cache: cliCache,
+    cachePath: cliCachePath,
+    noCache: cliNoCache,
+    tsConfigCache: configCacheEnabled,
+    tsConfigCachePath: configCachePath,
     // Boolean OR: config `true` cannot be overridden to `false` from CLI.
     // Intentional — there are no --no-verbose / --no-keep-workspaces flags.
     // Precedence: CLI > YAML config > TS config
@@ -1311,13 +1316,15 @@ export async function runEvalCommand(
     cliCache: options.cache,
     cliNoCache: options.noCache,
     yamlCache: yamlCacheEnabled,
+    tsConfigCache: options.tsConfigCache,
   });
+  const activeCachePath = options.cachePath ?? yamlCachePath ?? options.tsConfigCachePath;
   const cache = cacheEnabled
-    ? new ResponseCache(yamlCachePath ? path.resolve(yamlCachePath) : undefined)
+    ? new ResponseCache(activeCachePath ? path.resolve(activeCachePath) : undefined)
     : undefined;
 
-  if (cacheEnabled) {
-    console.log(`Response cache: enabled${yamlCachePath ? ` (${yamlCachePath})` : ''}`);
+  if (cache) {
+    console.log(`Response cache: enabled (${cache.cachePath})`);
   }
 
   // Resolve suite-level threshold: CLI --threshold takes precedence over YAML execution.threshold.

apps/cli/test/eval.integration.test.ts

@@ -199,6 +199,23 @@ async function readDiagnostics(fixture: EvalFixture): Promise<Record<string, unk
   throw new Error(`Missing diagnostics file: ${fixture.diagnosticsPath}`);
 }
 
+async function writeTsCacheConfig(fixture: EvalFixture, cachePath: string): Promise<void> {
+  await writeFile(
+    path.join(fixture.suiteDir, 'agentv.config.ts'),
+    `export default { cache: { enabled: true, path: ${JSON.stringify(cachePath)} } };\n`,
+    'utf8',
+  );
+}
+
+async function prependYamlCacheConfig(fixture: EvalFixture, cachePath: string): Promise<void> {
+  const original = await readFile(fixture.testFilePath, 'utf8');
+  await writeFile(
+    fixture.testFilePath,
+    `execution:\n  cache: true\n  cache_path: ${cachePath}\n\n${original}`,
+    'utf8',
+  );
+}
+
 describe('agentv eval CLI', () => {
   it('writes results, summary, and prompt dumps using default directories', async () => {
     const fixture = await createFixture();
@@ -285,4 +302,117 @@ describe('agentv eval CLI', () => {
       await rm(fixture.baseDir, { recursive: true, force: true });
     }
   }, 30_000);
+
+  it('honors agentv.config.ts cache.path when response cache is enabled there', async () => {
+    const fixture = await createFixture();
+    try {
+      const cachePath = '.agentv/ts-response-cache';
+      await writeTsCacheConfig(fixture, cachePath);
+
+      const { stdout } = await runCli(fixture, ['eval', fixture.testFilePath]);
+
+      const resolvedCachePath = path.resolve(fixture.suiteDir, cachePath);
+      expect(stdout).toContain(`Response cache: enabled (${resolvedCachePath})`);
+      const diagnostics = await readDiagnostics(fixture);
+      expect(diagnostics).toMatchObject({
+        hasCache: true,
+        cachePath: resolvedCachePath,
+        useCache: true,
+      });
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
+  it('honors eval YAML execution.cache_path', async () => {
+    const fixture = await createFixture();
+    try {
+      const cachePath = '.agentv/yaml-response-cache';
+      await prependYamlCacheConfig(fixture, cachePath);
+
+      const { stdout } = await runCli(fixture, ['eval', fixture.testFilePath]);
+
+      const resolvedCachePath = path.resolve(fixture.suiteDir, cachePath);
+      expect(stdout).toContain(`Response cache: enabled (${resolvedCachePath})`);
+      const diagnostics = await readDiagnostics(fixture);
+      expect(diagnostics).toMatchObject({
+        hasCache: true,
+        cachePath: resolvedCachePath,
+        useCache: true,
+      });
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
+  it('honors CLI --cache-path as an explicit response cache opt-in', async () => {
+    const fixture = await createFixture();
+    try {
+      const cachePath = '.agentv/cli-response-cache';
+
+      const { stdout } = await runCli(fixture, [
+        'eval',
+        fixture.testFilePath,
+        '--cache-path',
+        cachePath,
+      ]);
+
+      const resolvedCachePath = path.resolve(fixture.suiteDir, cachePath);
+      expect(stdout).toContain(`Response cache: enabled (${resolvedCachePath})`);
+      const diagnostics = await readDiagnostics(fixture);
+      expect(diagnostics).toMatchObject({
+        hasCache: true,
+        cachePath: resolvedCachePath,
+        useCache: true,
+      });
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
+  it('lets --no-cache override config-driven response cache settings', async () => {
+    const fixture = await createFixture();
+    try {
+      await writeTsCacheConfig(fixture, '.agentv/disabled-response-cache');
+
+      const { stdout } = await runCli(fixture, ['eval', fixture.testFilePath, '--no-cache']);
+
+      expect(stdout).not.toContain('Response cache: enabled');
+      const diagnostics = await readDiagnostics(fixture);
+      expect(diagnostics).toMatchObject({
+        hasCache: false,
+        cachePath: null,
+        useCache: false,
+      });
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
+  it('keeps response cache help separate from transcript replay terminology', async () => {
+    const result = await execa('bun', ['--no-env-file', CLI_ENTRY, 'eval', 'run', '--help'], {
+      cwd: projectRoot,
+      env: { ...process.env, CI: 'true' },
+      reject: false,
+    });
+    const helpText = `${result.stdout}\n${result.stderr}`;
+    expect(helpText).toContain('--cache');
+    expect(helpText).toContain('--cache-path');
+    expect(helpText).toContain('--transcript');
+
+    const cacheHelp = helpText
+      .split(/\r?\n/)
+      .filter((line) => line.includes('--cache'))
+      .join('\n')
+      .toLowerCase();
+    expect(cacheHelp).toContain('response cache');
+    expect(cacheHelp).not.toContain('replay');
+
+    const transcriptHelp = helpText
+      .split(/\r?\n/)
+      .filter((line) => line.includes('--transcript'))
+      .join('\n')
+      .toLowerCase();
+    expect(transcriptHelp).not.toContain('cache');
+  }, 30_000);
 });

apps/cli/test/fixtures/mock-run-evaluation.ts

@@ -25,6 +25,14 @@ interface RunEvaluationOptionsLike {
   readonly onResult?: (result: EvaluationResultLike) => Promise<void> | void;
 }
 
+function getCachePath(cache: unknown): string | null {
+  if (!cache || typeof cache !== 'object') {
+    return null;
+  }
+  const maybeCachePath = (cache as { readonly cachePath?: unknown }).cachePath;
+  return typeof maybeCachePath === 'string' ? maybeCachePath : null;
+}
+
 interface EvaluationResultLike {
   readonly testId: string;
   readonly score: number;
@@ -82,6 +90,8 @@ async function maybeWriteDiagnostics(
     agentTimeoutMs: options.agentTimeoutMs ?? null,
     promptDumpDir: options.promptDumpDir,
     filter: options.filter ?? null,
+    hasCache: options.cache !== undefined,
+    cachePath: getCachePath(options.cache),
     useCache: options.useCache ?? false,
     envSample: process.env.CLI_ENV_SAMPLE ?? null,
     envRootOnly: process.env.CLI_ENV_ROOT_ONLY ?? null,

apps/web/src/content/docs/docs/evaluation/running-evals.mdx

@@ -427,6 +427,40 @@ The `{timestamp}` placeholder is replaced with an ISO-like timestamp (e.g., `202
 
 **Precedence:** CLI flags > project-local `.agentv/config.yaml` > home/global `$AGENTV_HOME/config.yaml` (or `~/.agentv/config.yaml`) > `agentv.config.ts` > built-in defaults.
 
+## Response Cache
+
+AgentV's response cache stores exact provider responses on disk to reduce repeated live LLM calls while iterating on the same eval. It is disabled by default. Enable it from the CLI with `--cache`, or enable it with a custom directory using `--cache-path`:
+
+```bash
+agentv eval evals/dataset.eval.yaml --cache
+agentv eval evals/dataset.eval.yaml --cache-path .agentv/response-cache
+```
+
+Eval YAML can enable the same cache per suite:
+
+```yaml
+execution:
+  cache: true
+  cache_path: .agentv/response-cache
+```
+
+Project TypeScript config can set the project default:
+
+```typescript
+import { defineConfig } from '@agentv/core';
+
+export default defineConfig({
+  cache: {
+    enabled: true,
+    path: '.agentv/response-cache',
+  },
+});
+```
+
+`--no-cache` disables response caching regardless of CLI, eval YAML, or TypeScript config. Cache path precedence is `--cache-path` > eval YAML `execution.cache_path` > TypeScript config `cache.path` > `.agentv/cache`.
+
+Response cache and replay are separate concepts. The response cache is an iteration aid for repeated live provider calls. Transcript or fixture replay is target substitution from curated artifacts, and graders still run fresh against the replayed output.
+
 ## Environment Variables
 
 ### AGENTV_HOME

packages/core/src/evaluation/cache/response-cache.ts

@@ -13,7 +13,7 @@ const DEFAULT_CACHE_PATH = '.agentv/cache';
  * Directory structure: <cache_path>/<first-2-chars>/<full-hash>.json
  */
 export class ResponseCache implements EvaluationCache {
-  private readonly cachePath: string;
+  readonly cachePath: string;
 
   constructor(cachePath?: string) {
     this.cachePath = cachePath ?? DEFAULT_CACHE_PATH;
@@ -47,16 +47,21 @@ export class ResponseCache implements EvaluationCache {
  *
  * Precedence:
  *   1. --no-cache CLI flag → always disabled
- *   2. --cache CLI flag OR execution.cache YAML → enabled
- *   3. Default → disabled (safe for variability testing)
+ *   2. --cache CLI flag → enabled
+ *   3. execution.cache YAML → enabled/disabled for that eval file
+ *   4. agentv.config.ts cache.enabled → project default
+ *   5. Default → disabled (safe for variability testing)
  */
 export function shouldEnableCache(params: {
   cliCache: boolean;
   cliNoCache: boolean;
   yamlCache?: boolean;
+  tsConfigCache?: boolean;
 }): boolean {
   if (params.cliNoCache) return false;
-  return params.cliCache || params.yamlCache === true;
+  if (params.cliCache) return true;
+  if (params.yamlCache !== undefined) return params.yamlCache;
+  return params.tsConfigCache === true;
 }
 
 /**

packages/core/src/evaluation/config.ts

@@ -64,7 +64,7 @@ const AgentVConfigSchema = z.object({
     .object({
       /** Enable response caching */
       enabled: z.boolean().optional(),
-      /** Cache file path */
+      /** Response cache directory */
       path: z.string().optional(),
     })
     .optional(),