fix(cli): make required_version advisory, warn on eval failure instead of prompting/exiting (#1396)

Author: christso

apps/cli/src/commands/eval/run-eval.ts

@@ -31,7 +31,11 @@ import {
   subscribeToPiLogEntries,
 } from '@agentv/core';
 
-import { enforceRequiredVersion } from '../../version-check.js';
+import {
+  type VersionCheckResult,
+  enforceRequiredVersion,
+  formatRequiredVersionFailureNote,
+} from '../../version-check.js';
 import {
   type RemoteExportStatus,
   getRelativeRunPath,
@@ -1107,9 +1111,11 @@ export async function runEvalCommand(
   // Pass a dummy file in cwd so the search starts from the working directory.
   const yamlConfig = await loadConfig(path.join(cwd, '_'), repoRoot);
 
-  // Check required_version before proceeding with eval
+  // Check required_version before proceeding with eval. A mismatch is advisory
+  // unless the user explicitly opts into --strict.
+  let requiredVersionCheck: VersionCheckResult | undefined;
   if (yamlConfig?.required_version) {
-    await enforceRequiredVersion(yamlConfig.required_version, {
+    requiredVersionCheck = await enforceRequiredVersion(yamlConfig.required_version, {
       strict: normalizeBoolean(input.rawOptions.strict),
     });
   }
@@ -1774,6 +1780,13 @@ export async function runEvalCommand(
       resolvedThreshold !== undefined ? { threshold: resolvedThreshold } : undefined;
     const summary = calculateEvaluationSummary(summaryResults, thresholdOpts);
     console.log(formatEvaluationSummary(summary, thresholdOpts));
+    if (
+      requiredVersionCheck &&
+      !requiredVersionCheck.satisfied &&
+      (summary.qualityFailureCount > 0 || summary.executionErrorCount > 0)
+    ) {
+      console.log(`\n${formatRequiredVersionFailureNote(requiredVersionCheck)}`);
+    }
 
     // Exit code: 2 when all tests are execution errors (no evaluation performed),
     // 1 when any test scored below threshold.

apps/cli/src/commands/results/serve.ts

@@ -24,9 +24,9 @@
  * variants. They share handler functions via DataContext, differing only in
  * how searchDir is resolved.
  *
- * Before starting the server, the command enforces `required_version` from
- * the cwd's `.agentv/config.yaml` (single-project scope) via
- * `enforceRequiredVersion()`, matching the behavior of `agentv eval`.
+ * Before starting the server, the command checks `required_version` from
+ * the cwd's `.agentv/config.yaml` (single-project scope) and warns on
+ * mismatches without prompting, self-updating, or blocking startup.
  *
  * Exported functions (for testing):
  *   - resolveSourceFile(source, cwd) — resolves a run manifest path
@@ -2097,11 +2097,11 @@ export const resultsServeCommand = command({
     }
 
     // ── Version check ────────────────────────────────────────────────
-    // Enforce `required_version` from .agentv/config.yaml so Dashboard/serve
-    // match `agentv eval` behavior. Same prompt in TTY, warn+continue
-    // otherwise. Single-project scope only — when one agentv instance
-    // serves multiple repos with differing version requirements, a
-    // per-project local install is required instead.
+    // Check `required_version` from .agentv/config.yaml so Dashboard/serve
+    // match `agentv eval` behavior. Version mismatches are advisory by
+    // default. Single-project scope only — when one agentv instance serves
+    // multiple repos with differing version requirements, a per-project local
+    // install is required instead.
     const repoRoot = await findRepoRoot(cwd);
     const yamlConfig = await loadConfig(path.join(cwd, '_'), repoRoot);
     if (yamlConfig?.required_version) {

apps/cli/src/self-update.ts

@@ -1,16 +1,12 @@
 /**
  * Shared self-update logic for agentv.
  *
- * Used by both `agentv self update` and the version-check prompt
- * when the installed version doesn't satisfy `required_version`.
+ * Used only by the explicit `agentv self update` command. Project
+ * `required_version` checks are advisory and never invoke this module.
  *
- * When called from the version-check prompt, a `versionRange` (from the
- * project's `required_version` config) is passed through as the npm/bun
- * version specifier (e.g., `agentv@">=4.1.0"`). This ensures the update
- * respects the project's constraints and avoids unintended major-version jumps.
- *
- * When called from `agentv self update` (no range), it installs `@latest` for stable
- * versions or `@next` when the current version has a prerelease identifier.
+ * By default it installs `@latest` for stable versions or `@next` when the
+ * current version has a prerelease identifier. Callers may pass a
+ * `versionRange`/dist tag when they need a specific install specifier.
  *
  * Install scope detection: if `process.argv[1]` contains `node_modules`,
  * agentv was invoked from a local project dependency (e.g. `npx agentv` or

apps/cli/src/version-check.ts

@@ -1,11 +1,9 @@
-import { coerce, major, satisfies, validRange } from 'semver';
+import { coerce, satisfies, validRange } from 'semver';
 
 import packageJson from '../package.json' with { type: 'json' };
-import { performSelfUpdate } from './self-update.js';
 
 const ANSI_YELLOW = '\u001b[33m';
 const ANSI_RED = '\u001b[31m';
-const ANSI_GREEN = '\u001b[32m';
 const ANSI_RESET = '\u001b[0m';
 
 export interface VersionCheckResult {
@@ -15,8 +13,12 @@ export interface VersionCheckResult {
 }
 
 /**
- * Validate and check the installed version against a required semver range.
- * Throws on malformed range strings.
+ * Advisory version checks for project `required_version`.
+ *
+ * A mismatched installed version never prompts, self-updates, or exits by
+ * default. Commands may print the returned warning to help users diagnose
+ * failures, while explicit `--strict` callers can still opt into a hard gate.
+ * Malformed ranges remain configuration errors.
  */
 export function checkVersion(requiredVersion: string): VersionCheckResult {
   const currentVersion = packageJson.version;
@@ -34,23 +36,30 @@ export function checkVersion(requiredVersion: string): VersionCheckResult {
   };
 }
 
+function formatVersionMismatch(result: VersionCheckResult): string {
+  return `agentv ${result.currentVersion} does not satisfy this project's required_version ${result.requiredRange}`;
+}
+
+export function formatRequiredVersionWarning(result: VersionCheckResult): string {
+  return `${ANSI_YELLOW}Warning: ${formatVersionMismatch(result)}. Run \`agentv self update\`.${ANSI_RESET}`;
+}
+
+export function formatRequiredVersionFailureNote(result: VersionCheckResult): string {
+  return `note: ${formatVersionMismatch(result)} - this may be the cause. Run \`agentv self update\`.`;
+}
+
 /**
- * Run the version compatibility check and handle user interaction.
+ * Run the version compatibility check.
  *
- * - If the version satisfies the range, returns silently.
+ * - If the version satisfies the range, returns the satisfied result silently.
  * - If the range is malformed, prints an error and exits with code 1.
- * - If the version is below the range:
- *   - Interactive (TTY): warns and prompts "Update now? (Y/n)".
- *     Y → runs self-update inline (constrained to the config range),
- *         then exits with a message to re-run the command.
- *     N → continues the command as-is.
- *   - Non-interactive: warns to stderr, continues (unless strict).
- *   - Strict mode: warns and exits with code 1.
+ * - If the version is below the range, warns to stderr and continues.
+ * - Strict mode remains an explicit opt-in hard failure.
  */
-export async function enforceRequiredVersion(
+export function enforceRequiredVersion(
   requiredVersion: string,
   options?: { strict?: boolean },
-): Promise<void> {
+): VersionCheckResult {
   let result: VersionCheckResult;
   try {
     result = checkVersion(requiredVersion);
@@ -60,59 +69,19 @@ export async function enforceRequiredVersion(
   }
 
   if (result.satisfied) {
-    return;
+    return result;
   }
 
-  const warning = `${ANSI_YELLOW}Warning: This project requires agentv ${result.requiredRange} but you have ${result.currentVersion}.${ANSI_RESET}`;
+  const warning = formatRequiredVersionWarning(result);
 
   if (options?.strict) {
-    console.error(`${warning}\n  Run \`agentv self update\` to upgrade.`);
+    console.error(warning);
     console.error(
       `${ANSI_RED}Aborting: --strict mode requires the installed version to satisfy the required range.${ANSI_RESET}`,
     );
     process.exit(1);
   }
 
-  if (process.stdin.isTTY && process.stdout.isTTY) {
-    console.warn(warning);
-    const shouldUpdate = await promptUpdate();
-    if (shouldUpdate) {
-      await runInlineUpdate(result.currentVersion, result.requiredRange);
-    }
-    // N → continue the command without interruption
-  } else {
-    // Non-interactive: warn to stderr and continue
-    process.stderr.write(`${warning}\n  Run \`agentv self update\` to upgrade.\n`);
-  }
-}
-
-async function promptUpdate(): Promise<boolean> {
-  const { confirm } = await import('@inquirer/prompts');
-  return confirm({ message: 'Update now?', default: true });
-}
-
-async function runInlineUpdate(currentVersion: string, versionRange: string): Promise<void> {
-  // Cap at the current major version to avoid unintended breaking changes.
-  // e.g., if current is 4.14.2 and range is ">=4.1.0", install ">=4.1.0 <5.0.0"
-  // so that a hypothetical 5.0.0 is never pulled in by auto-update.
-  const currentMajor = major(coerce(currentVersion) ?? currentVersion);
-  const safeRange = `${versionRange} <${currentMajor + 1}.0.0`;
-
-  console.log('');
-  const result = await performSelfUpdate({ currentVersion, versionRange: safeRange });
-
-  if (!result.success) {
-    console.error(`${ANSI_RED}Update failed. Run \`agentv self update\` manually.${ANSI_RESET}`);
-    process.exit(1);
-  }
-
-  if (result.newVersion) {
-    console.log(
-      `\n${ANSI_GREEN}Update complete: ${currentVersion} → ${result.newVersion}${ANSI_RESET}`,
-    );
-  } else {
-    console.log(`\n${ANSI_GREEN}Update complete.${ANSI_RESET}`);
-  }
-  console.log('Please re-run your command.');
-  process.exit(0);
+  process.stderr.write(`${warning}\n`);
+  return result;
 }

apps/cli/test/eval.integration.test.ts

@@ -4,6 +4,7 @@ import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { execa } from 'execa';
+import packageJson from '../package.json' with { type: 'json' };
 import { assertCoreBuild } from './setup-core-build.js';
 
 assertCoreBuild();
@@ -215,6 +216,17 @@ async function writeTsOutputConfig(fixture: EvalFixture, outputDir: string): Pro
   );
 }
 
+async function writeRequiredVersionConfig(
+  fixture: EvalFixture,
+  requiredVersion: string,
+): Promise<void> {
+  await writeFile(
+    path.join(fixture.suiteDir, '.agentv', 'config.yaml'),
+    `required_version: "${requiredVersion}"\n`,
+    'utf8',
+  );
+}
+
 async function expectFileExists(filePath: string): Promise<void> {
   await access(filePath);
 }
@@ -262,6 +274,58 @@ describe('agentv eval CLI', () => {
     }
   }, 30_000);
 
+  it('surfaces required_version mismatch note when the eval fails', async () => {
+    const fixture = await createFixture();
+    try {
+      await writeRequiredVersionConfig(fixture, '>=999.0.0');
+
+      const { stdout, stderr, exitCode } = await runCli(fixture, [
+        'eval',
+        fixture.testFilePath,
+        '--threshold',
+        '0.8',
+      ]);
+
+      expect(exitCode).toBe(1);
+      expect(stdout).toContain('RESULT: FAIL');
+      expect(stdout).toContain(
+        `note: agentv ${packageJson.version} does not satisfy this project's required_version >=999.0.0 - this may be the cause. Run \`agentv self update\`.`,
+      );
+      expect(stderr).toContain(
+        `Warning: agentv ${packageJson.version} does not satisfy this project's required_version >=999.0.0. Run \`agentv self update\`.`,
+      );
+      expect(`${stdout}\n${stderr}`).not.toContain('Update now?');
+      expect(`${stdout}\n${stderr}`).not.toContain('Update complete');
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
+  it('keeps required_version mismatch low-noise when the eval passes', async () => {
+    const fixture = await createFixture();
+    try {
+      await writeRequiredVersionConfig(fixture, '>=999.0.0');
+
+      const { stdout, stderr, exitCode } = await runCli(fixture, [
+        'eval',
+        fixture.testFilePath,
+        '--threshold',
+        '0.5',
+      ]);
+
+      expect(exitCode).toBe(0);
+      expect(stdout).toContain('RESULT: PASS');
+      expect(stdout).not.toContain('note: agentv');
+      expect(stderr).toContain(
+        `Warning: agentv ${packageJson.version} does not satisfy this project's required_version >=999.0.0. Run \`agentv self update\`.`,
+      );
+      expect(`${stdout}\n${stderr}`).not.toContain('Update now?');
+      expect(`${stdout}\n${stderr}`).not.toContain('Update complete');
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
   it('writes canonical artifacts under an explicit --output directory', async () => {
     const fixture = await createFixture();
     try {

apps/cli/test/version-check.test.ts

@@ -1,6 +1,10 @@
-import { describe, expect, test } from 'bun:test';
+import { describe, expect, spyOn, test } from 'bun:test';
 import packageJson from '../package.json' with { type: 'json' };
-import { checkVersion } from '../src/version-check.js';
+import {
+  checkVersion,
+  enforceRequiredVersion,
+  formatRequiredVersionFailureNote,
+} from '../src/version-check.js';
 
 const [major, minor] = packageJson.version.split('.').map(Number);
 
@@ -50,3 +54,53 @@ describe('checkVersion', () => {
     expect(result.currentVersion).toMatch(/^\d+\.\d+\.\d+/);
   });
 });
+
+describe('enforceRequiredVersion', () => {
+  test('warns and continues on mismatch by default', () => {
+    const stderrSpy = spyOn(process.stderr, 'write').mockImplementation(() => true);
+    const exitSpy = spyOn(process, 'exit').mockImplementation((() => {
+      throw new Error('process.exit should not be called');
+    }) as never);
+
+    try {
+      const result = enforceRequiredVersion('>=99.0.0');
+
+      expect(result.satisfied).toBe(false);
+      expect(result.requiredRange).toBe('>=99.0.0');
+      expect(exitSpy).not.toHaveBeenCalled();
+      expect(String(stderrSpy.mock.calls[0]?.[0] ?? '')).toContain(
+        `agentv ${packageJson.version} does not satisfy this project's required_version >=99.0.0`,
+      );
+      expect(String(stderrSpy.mock.calls[0]?.[0] ?? '')).toContain('agentv self update');
+      expect(String(stderrSpy.mock.calls[0]?.[0] ?? '')).not.toContain('Update now?');
+    } finally {
+      stderrSpy.mockRestore();
+      exitSpy.mockRestore();
+    }
+  });
+
+  test('keeps strict mode as an explicit hard failure', () => {
+    const errorSpy = spyOn(console, 'error').mockImplementation(() => {});
+    const exitSpy = spyOn(process, 'exit').mockImplementation(((code?: number) => {
+      throw new Error(`process.exit(${code})`);
+    }) as never);
+
+    try {
+      expect(() => enforceRequiredVersion('>=99.0.0', { strict: true })).toThrow('process.exit(1)');
+      expect(exitSpy).toHaveBeenCalledWith(1);
+    } finally {
+      errorSpy.mockRestore();
+      exitSpy.mockRestore();
+    }
+  });
+});
+
+describe('formatRequiredVersionFailureNote', () => {
+  test('formats the eval failure diagnostic note', () => {
+    const result = checkVersion('>=99.0.0');
+
+    expect(formatRequiredVersionFailureNote(result)).toBe(
+      `note: agentv ${packageJson.version} does not satisfy this project's required_version >=99.0.0 - this may be the cause. Run \`agentv self update\`.`,
+    );
+  });
+});

apps/web/src/content/docs/docs/evaluation/running-evals.mdx

@@ -411,11 +411,16 @@ The value is a **semver range** using standard npm syntax (e.g., `>=2.12.0`, `^2
 | Condition | Interactive (TTY) | Non-interactive (CI) |
 |-----------|-------------------|---------------------|
 | Version satisfies range | Runs silently | Runs silently |
-| Version below range | Warns + prompts to continue | Warns to stderr, continues |
+| Version below range | Warns to stderr, continues | Warns to stderr, continues |
 | `--strict` flag + mismatch | Warns + exits 1 | Warns + exits 1 |
 | No `required_version` set | Runs silently | Runs silently |
 | Malformed semver range | Error + exits 1 | Error + exits 1 |
 
+By default, `required_version` is advisory: AgentV never prompts, self-updates,
+or blocks a run just because the installed version is outside the range. If an
+eval fails or has execution errors while the range is unsatisfied, the summary
+includes a note that the version mismatch may be the cause.
+
 Use `--strict` in CI pipelines to enforce version requirements:
 
 ```bash