feat(vitest-plugin): support vitest 5 alongside 3 and 4

Vitest 5 reworked the benchmark backend: the dedicated `NodeBenchmarkRunner`
and the `vitest/runners` / `vitest/suite` entrypoints are gone, benchmarks now
run inside `test()` through the unified `TestRunner`, and tinybench moved to v6
(stats moved from `result.benchmark` to `task.result.latency`, `includeSamples`
became `retainSamples`).

Detect the installed Vitest generation and select the integration seam behind a
`VitestBackend` abstraction so the rest of the plugin never inspects the version:
- v3/4 keep the custom benchmark runner per instrument mode (analysis/walltime).
- v5 installs instrumentation from a setup file that patches the shared
  `TestRunner.runBenchmarks` static. A setup file (rather than a custom
  `test.runner`) leaves the runner untouched for non-benchmark tests and also
  applies to the browser pool.

Author: GuillaumeLagrange

packages/vitest-plugin/package.json

@@ -41,13 +41,13 @@
   "peerDependencies": {
     "tinybench": ">=2.9.0",
     "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0",
-    "vitest": "^3.2 || ^4"
+    "vitest": "^3.2 || ^4 || ^5.0.0-beta"
   },
   "devDependencies": {
     "@total-typescript/shoehorn": "^0.1.1",
     "execa": "^8.0.1",
     "tinybench": "^2.9.0",
     "vite": "^7.0.0",
-    "vitest": "^4.0.18"
+    "vitest": "5.0.0-beta.5"
   }
 }

packages/vitest-plugin/rollup.config.ts

@@ -20,16 +20,24 @@ export default defineConfig([
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/],
   },
+  // The built layout mirrors the source layout (dist/legacy/*, dist/v5/*) so the
+  // plugin resolves the seam files with one path rule in both dev and prod.
   {
-    input: "src/analysis.ts",
-    output: { file: "dist/analysis.mjs", format: "es" },
+    input: "src/legacy/analysis.ts",
+    output: { file: "dist/legacy/analysis.mjs", format: "es" },
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/],
   },
   {
-    input: "src/walltime/index.ts",
-    output: { file: "dist/walltime.mjs", format: "es" },
+    input: "src/legacy/walltime.ts",
+    output: { file: "dist/legacy/walltime.mjs", format: "es" },
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/],
   },
+  {
+    input: "src/v5/setup.ts",
+    output: { file: "dist/v5/setup.mjs", format: "es" },
+    plugins: jsPlugins(pkg.version),
+    external: ["@codspeed/core", /^vitest/, "tinybench"],
+  },
 ]);

packages/vitest-plugin/src/__tests__/index.test.ts

@@ -43,6 +43,19 @@ vi.mock("fs", () => {
 
 console.warn = vi.fn();
 
+const EXPECTED_EXEC_ARGV = [
+  "--interpreted-frames-native-stack",
+  "--allow-natives-syntax",
+  "--hash-seed=1",
+  "--random-seed=1",
+  "--no-opt",
+  "--predictable",
+  "--predictable-gc-schedule",
+  "--expose-gc",
+  "--no-concurrent-sweeping",
+  "--max-old-space-size=4096",
+];
+
 describe("codSpeedPlugin", () => {
   beforeAll(() => {
     // Set environment variables to trigger instrumented mode
@@ -54,6 +67,7 @@ describe("codSpeedPlugin", () => {
     // Clean up environment variables
     delete process.env.CODSPEED_ENV;
     delete process.env.CODSPEED_RUNNER_MODE;
+    fsMocks.setMockVersion("4.0.18");
   });
 
   it("should have a name", async () => {
@@ -65,7 +79,9 @@ describe("codSpeedPlugin", () => {
   });
 
   describe("apply", () => {
-    it("should not apply the plugin when the mode is not benchmark", async () => {
+    it("should not apply the plugin when the mode is not benchmark (v3/v4)", async () => {
+      fsMocks.setMockVersion("4.0.18");
+
       const applyPlugin = applyPluginFunction(
         {},
         fromPartial({ mode: "test" }),
@@ -74,7 +90,8 @@ describe("codSpeedPlugin", () => {
       expect(applyPlugin).toBe(false);
     });
 
-    it("should apply the plugin when there is no instrumentation", async () => {
+    it("should apply the plugin when there is no instrumentation (v3/v4)", async () => {
+      fsMocks.setMockVersion("4.0.18");
       coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(false);
 
       const applyPlugin = applyPluginFunction(
@@ -88,7 +105,8 @@ describe("codSpeedPlugin", () => {
       expect(applyPlugin).toBe(true);
     });
 
-    it("should apply the plugin when there is instrumentation", async () => {
+    it("should apply the plugin when there is instrumentation (v3/v4)", async () => {
+      fsMocks.setMockVersion("4.0.18");
       coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true);
 
       const applyPlugin = applyPluginFunction(
@@ -98,51 +116,60 @@ describe("codSpeedPlugin", () => {
 
       expect(applyPlugin).toBe(true);
     });
+
+    it("should stay active regardless of mode on v5 (benchmark gating happens in config)", async () => {
+      fsMocks.setMockVersion("5.0.0-beta.5");
+      coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true);
+
+      const applyPlugin = applyPluginFunction(
+        {},
+        fromPartial({ mode: "test" }),
+      );
+
+      expect(applyPlugin).toBe(true);
+      fsMocks.setMockVersion("4.0.18");
+    });
   });
 
   it("should apply the codspeed config for v4", () => {
+    fsMocks.setMockVersion("4.0.18");
     const config = resolvedCodSpeedPlugin.config;
     if (typeof config !== "function")
       throw new Error("config is not a function");
 
-    const result = config.call({} as never, {}, fromPartial({}));
+    const result = config.call(
+      {} as never,
+      {},
+      fromPartial({ mode: "benchmark" }),
+    );
 
     expect(result).toStrictEqual({
       test: {
         globalSetup: [
           expect.stringContaining("packages/vitest-plugin/src/globalSetup.ts"),
         ],
         pool: "forks",
-        execArgv: [
-          "--interpreted-frames-native-stack",
-          "--allow-natives-syntax",
-          "--hash-seed=1",
-          "--random-seed=1",
-          "--no-opt",
-          "--predictable",
-          "--predictable-gc-schedule",
-          "--expose-gc",
-          "--no-concurrent-sweeping",
-          "--max-old-space-size=4096",
-        ],
+        execArgv: EXPECTED_EXEC_ARGV,
         runner: expect.stringContaining(
-          "packages/vitest-plugin/src/analysis.ts",
+          "packages/vitest-plugin/src/legacy/analysis.ts",
         ),
       },
     });
   });
 
   it("should apply the codspeed config for v3 with poolOptions", () => {
-    // Set mock version to v3
     fsMocks.setMockVersion("3.2.0");
 
-    // Create a new plugin instance to pick up the mocked version
     const v3Plugin = codspeedPlugin();
     const config = v3Plugin.config;
     if (typeof config !== "function")
       throw new Error("config is not a function");
 
-    const result = config.call({} as never, {}, fromPartial({}));
+    const result = config.call(
+      {} as never,
+      {},
+      fromPartial({ mode: "benchmark" }),
+    );
 
     expect(result).toStrictEqual({
       test: {
@@ -152,27 +179,70 @@ describe("codSpeedPlugin", () => {
         pool: "forks",
         poolOptions: {
           forks: {
-            execArgv: [
-              "--interpreted-frames-native-stack",
-              "--allow-natives-syntax",
-              "--hash-seed=1",
-              "--random-seed=1",
-              "--no-opt",
-              "--predictable",
-              "--predictable-gc-schedule",
-              "--expose-gc",
-              "--no-concurrent-sweeping",
-              "--max-old-space-size=4096",
-            ],
+            execArgv: EXPECTED_EXEC_ARGV,
           },
         },
         runner: expect.stringContaining(
-          "packages/vitest-plugin/src/analysis.ts",
+          "packages/vitest-plugin/src/legacy/analysis.ts",
         ),
       },
     });
 
-    // Reset mock version back to v4
     fsMocks.setMockVersion("4.0.18");
   });
+
+  describe("v5 config", () => {
+    it("should not inject config when benchmarks are not enabled", () => {
+      fsMocks.setMockVersion("5.0.0-beta.5");
+      const v5Plugin = codspeedPlugin();
+      const config = v5Plugin.config;
+      if (typeof config !== "function")
+        throw new Error("config is not a function");
+
+      const result = config.call(
+        {} as never,
+        {},
+        fromPartial({ mode: "test" }),
+      );
+
+      expect(result).toBeUndefined();
+      fsMocks.setMockVersion("4.0.18");
+    });
+
+    it("should inject the v5 setup file (not a runner) when benchmarks are enabled", () => {
+      fsMocks.setMockVersion("5.0.0-beta.5");
+      const v5Plugin = codspeedPlugin();
+      const config = v5Plugin.config;
+      if (typeof config !== "function")
+        throw new Error("config is not a function");
+
+      const result = config.call(
+        {} as never,
+        // `benchmark.enabled` is a Vitest 5 config field the v3/4 typings (which
+        // this file may be compiled against) don't expose.
+        { test: { benchmark: { enabled: true } } } as never,
+        fromPartial({ mode: "test" }),
+      );
+
+      expect(result).toStrictEqual({
+        test: {
+          globalSetup: [
+            expect.stringContaining(
+              "packages/vitest-plugin/src/globalSetup.ts",
+            ),
+          ],
+          pool: "forks",
+          execArgv: EXPECTED_EXEC_ARGV,
+          setupFiles: [
+            expect.stringContaining("packages/vitest-plugin/src/v5/setup.ts"),
+          ],
+        },
+      });
+      // The v5 path must not set a custom runner.
+      expect(
+        (result as { test?: { runner?: unknown } })?.test?.runner,
+      ).toBeUndefined();
+      fsMocks.setMockVersion("4.0.18");
+    });
+  });
 });

packages/vitest-plugin/src/__tests__/instrumented.test.ts

@@ -1,7 +1,15 @@
 import { fromPartial } from "@total-typescript/shoehorn";
 import { describe, expect, it, vi, type RunnerTestSuite } from "vitest";
+// `vitest/suite` only exists on Vitest 3/4; this file is excluded from the test
+// run under v5+ (see vitest.config.ts).
+// eslint-disable-next-line import/no-unresolved
 import { getBenchFn } from "vitest/suite";
-import { AnalysisRunner as CodSpeedRunner } from "../analysis";
+import { AnalysisRunner as CodSpeedRunner } from "../legacy/analysis";
+
+// The legacy AnalysisRunner targets the Vitest 3/4 benchmark backend
+// (`NodeBenchmarkRunner`, `vitest/suite`), which Vitest 5 removed. This whole
+// file is excluded from the test run under v5+ (see vitest.config.ts); the v5
+// path is covered separately.
 
 const coreMocks = vi.hoisted(() => {
   return {

packages/vitest-plugin/src/index.ts

@@ -7,49 +7,33 @@ import {
   SetupInstrumentsRequestBody,
   SetupInstrumentsResponse,
 } from "@codspeed/core";
-import { readFileSync } from "fs";
-import { createRequire } from "module";
 import { join } from "path";
 import { Plugin } from "vite";
 import { type ViteUserConfig } from "vitest/config";
+import { resolveVitestBackend } from "./vitestBackend";
 
 // get this file's directory path from import.meta.url
 const __dirname = new URL(".", import.meta.url).pathname;
 const isFileInTs = import.meta.url.endsWith(".ts");
 
-function getCodSpeedFileFromName(name: string) {
+/**
+ * Resolve a plugin-owned file (globalSetup, seam entry points) shipped alongside
+ * this module. Source (`.ts`) and built (`.mjs`) layouts are kept identical (see
+ * rollup.config.ts), so the same relative `name` works in both.
+ */
+function resolveFile(name: string): string {
   const fileExtension = isFileInTs ? "ts" : "mjs";
-
   return join(__dirname, `${name}.${fileExtension}`);
 }
 
-function getVitestMajorVersion(): number | null {
-  try {
-    // Resolve vitest from the project's perspective (cwd), not from the plugin's location
-    // This ensures we detect the vitest version the user has installed
-    const require = createRequire(join(process.cwd(), "package.json"));
-    const vitestPkgPath = require.resolve("vitest/package.json");
-    const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8"));
-    return parseInt(vitestPkg.version.split(".")[0], 10);
-  } catch {
-    return null;
-  }
-}
-
-function getRunnerFile(): string | undefined {
-  const instrumentMode = getInstrumentMode();
-  if (instrumentMode === "disabled") {
-    return undefined;
-  }
-
-  return getCodSpeedFileFromName(instrumentMode);
-}
-
 export default function codspeedPlugin(): Plugin {
+  // Resolved lazily on each hook rather than once here: the installed Vitest
+  // version is detected from the project's cwd, which isn't reliably knowable at
+  // plugin-construction time (and tests swap it between construction and use).
   return {
     name: "codspeed:vitest",
     apply(_, { mode }) {
-      if (mode !== "benchmark") {
+      if (!resolveVitestBackend().isActiveForViteMode(mode)) {
         return false;
       }
       if (
@@ -61,37 +45,19 @@ export default function codspeedPlugin(): Plugin {
       return true;
     },
     enforce: "post",
-    config(): ViteUserConfig {
-      const runnerFile = getRunnerFile();
-      const runnerMode = getCodspeedRunnerMode();
-      const v8Flags = getV8Flags();
-      const vitestMajorVersion = getVitestMajorVersion();
-      // by default, assume Vitest v4 or higher
-      const isVitestV4OrHigher = (vitestMajorVersion ?? 4) >= 4;
+    config(incomingConfig, { mode }): ViteUserConfig | undefined {
+      const backend = resolveVitestBackend();
+      if (!backend.isBenchmarkRun(incomingConfig, mode)) {
+        return undefined;
+      }
 
       const config: ViteUserConfig = {
         test: {
           pool: "forks",
-          ...(isVitestV4OrHigher
-            ? { execArgv: v8Flags }
-            : {
-                // Compat with Vitest v3
-                // See: https://vitest.dev/guide/migration.html#pool-rework
-                // poolOptions only exists in Vitest v3
-                poolOptions: {
-                  forks: {
-                    execArgv: v8Flags,
-                  },
-                },
-              }),
-          globalSetup: [getCodSpeedFileFromName("globalSetup")],
-          ...(runnerFile && {
-            runner: runnerFile,
-          }),
-          ...(runnerMode === "walltime" && {
-            benchmark: {
-              includeSamples: true,
-            },
+          globalSetup: [resolveFile("globalSetup")],
+          ...backend.getBenchmarkTestConfig(getV8Flags(), resolveFile),
+          ...(getCodspeedRunnerMode() === "walltime" && {
+            benchmark: backend.getWalltimeBenchmarkConfig(),
           }),
         },
       };