Added new test scripts for running and updating fixtures (#3213)

Author: AndreasArvidsson

package.json

@@ -12,7 +12,7 @@
   "scripts": {
     "build": "pnpm -r build",
     "clean": "pnpm -r clean",
-    "compile": "tsc --build && pnpm -r --if-present compile:esbuild",
+    "compile": "tsc --build && pnpm -r compile:esbuild",
     "fix:meta": "pnpm run meta-updater:base && pnpm -r exec prettier --write tsconfig.json package.json",
     "fix:syncpack": "syncpack fix",
     "fix:eslint": "pnpm lint:ts --fix",
@@ -27,10 +27,13 @@
     "preinstall": "npx only-allow pnpm",
     "test-compile": "tsc --build",
     "test": "pnpm -r test",
+    "test:subset": "pnpm -r test:subset",
+    "test:update": "pnpm -r test:update",
+    "test:update:subset": "pnpm -r test:update:subset",
     "generate-grammar": "pnpm -r generate-grammar",
     "transform-recorded-tests": "./packages/common/scripts/my-ts-node.js packages/cursorless-engine/src/scripts/transformRecordedTests/index.ts",
     "watch": "pnpm run -w --parallel '/^watch:.*/'",
-    "watch:esbuild": "pnpm run -r --parallel --if-present watch:esbuild",
+    "watch:esbuild": "pnpm run -r --parallel watch:esbuild",
     "watch:tsc": "tsc --build --watch"
   },
   "devDependencies": {

packages/cursorless-org-docs/src/docs/contributing/CONTRIBUTING.mdx

@@ -134,6 +134,8 @@ To run the specified subset of tests, say `"debug test subset"` (or use the "Run
 
 If you need to edit the subset of tests to run in the future, you can say `"debug edit subset"`.
 
+To update fixtures for the specified subset of tests from the command line, run `pnpm test:update:subset` from the repository root. To update fixtures for the full test suite instead, run `pnpm test:update`.
+
 ## Code formatting
 
 We use [`pre-commit`](https://pre-commit.com/) to automate autoformatting.

packages/cursorless-org-docs/src/docs/contributing/adding-a-new-scope.md

@@ -68,9 +68,9 @@ Then add parse tree patterns for the given scope to your language's `.scm` file
 
 The tests generated in step 4 only include the code example. Now that you've told Cursorless how to find the scope, we can automatically update the test cases to indicate where the scope should appear in your code examples.
 
-1. Say `"debug edit subset"` and alter the file to include just the name of your language
-2. Run the `Update fixtures subset` launch configuration to update your fixtures.
-3. Check that the fixtures now look as expected, and no other tests for your language have been altered. The VSCode source control side bar is useful for this purpose. For help understanding the scope test format, see the [scope test format docs](./scope-test-format.md)
+1. Say `"debug edit subset"` and alter the file to include just the name of your language.
+2. Run `pnpm test:update:subset` from the repository root to update the fixtures for the selected subset of tests. If you want to update all fixtures instead, run `pnpm test:update`.
+3. Check that the fixtures now look as expected, and no other tests for your language have been altered. The VSCode source control side bar is useful for this purpose. For help understanding the scope test format, see the [scope test format docs](./scope-test-format.md).
 
 ## 7. File a PR!
 

packages/cursorless-org-docs/src/docs/contributing/tests.md

@@ -20,6 +20,8 @@ You can get an overview of the various test contexts that exist locally by looki
 ## Common commands
 
 - `pnpm test` runs the default `test` script for each workspace package. In practice, this is the fast unit-oriented test pass.
+- `pnpm test:update` runs the test suite in fixture-update mode across the workspace.
+- `pnpm test:update:subset` runs fixture updates only for the tests selected in `packages/test-harness/testSubsetGrep.properties`.
 - `pnpm -F @cursorless/test-harness test:vscode` runs the VSCode test harness.
 - `pnpm -F @cursorless/test-harness test:neovim` runs the Neovim test harness.
 - `pnpm -F @cursorless/test-harness test:talonJs` runs the Talon-JS test harness.

packages/test-harness/package.json

@@ -13,11 +13,13 @@
     }
   },
   "scripts": {
-    "test": "env CURSORLESS_MODE=test my-ts-node src/scripts/runUnitTestsOnly.ts",
-    "test:subset": "env CURSORLESS_MODE=test env CURSORLESS_RUN_TEST_SUBSET=true my-ts-node src/scripts/runUnitTestsOnly.ts",
-    "test:vscode": "env CURSORLESS_MODE=test my-ts-node src/scripts/runVscodeTestsCI.ts",
-    "test:neovim": "env CURSORLESS_MODE=test my-ts-node src/scripts/runNeovimTestsCI.ts",
-    "test:talonJs": "env CURSORLESS_MODE=test my-ts-node src/scripts/runTalonJsTests.ts",
+    "test": "my-ts-node src/scripts/runUnitTestsOnly.ts",
+    "test:subset": "my-ts-node src/scripts/runUnitTestsOnly.ts --subset",
+    "test:update": "my-ts-node src/scripts/runUnitTestsOnly.ts --update",
+    "test:update:subset": "my-ts-node src/scripts/runUnitTestsOnly.ts --update --subset",
+    "test:vscode": "my-ts-node src/scripts/runVscodeTestsCI.ts",
+    "test:neovim": "my-ts-node src/scripts/runNeovimTestsCI.ts",
+    "test:talonJs": "my-ts-node src/scripts/runTalonJsTests.ts",
     "build:base": "esbuild --sourcemap --conditions=cursorless:bundler --bundle --external:vscode --external:./reporters/parallel-buffered --external:./worker.js --external:talon --format=cjs --platform=node",
     "build": "pnpm run build:runner:vscode && pnpm run build:runner:neovim && pnpm run build:tests && pnpm run build:unit && pnpm run build:talon && pnpm run build:talonJs",
     "build:runner:vscode": "pnpm run build:base ./src/runners/extensionTestsVscode.ts --outfile=dist/extensionTestsVscode.cjs",

packages/test-harness/src/scripts/runNeovimTestsCI.ts

@@ -4,6 +4,8 @@
 
 import { launchNeovimAndRunTests } from "../launchNeovimAndRunTests";
 
+process.env.CURSORLESS_MODE = "test";
+
 void (async () => {
   await launchNeovimAndRunTests();
 })();

packages/test-harness/src/scripts/runTalonJsTests.ts

@@ -1,9 +1,12 @@
 /**
  * Runs all Talon everywhere/JS tests.
  */
-import { TestType, runAllTests } from "../runAllTests";
+
+process.env.CURSORLESS_MODE = "test";
 
 void (async () => {
+  const { TestType, runAllTests } = await import("../runAllTests");
+
   try {
     await runAllTests(TestType.talonJs);
   } catch (_ex) {

packages/test-harness/src/scripts/runUnitTestsOnly.ts

@@ -1,6 +1,28 @@
 /**
  * Runs all tests that don't have to be run within VSCode.
  */
-import { TestType, runAllTests } from "../runAllTests";
 
-void runAllTests(TestType.unit);
+const supportedArgs = new Set(["--subset", "--update"]);
+
+const args = new Set(process.argv.slice(2));
+const unsupportedArgs = [...args].filter((arg) => !supportedArgs.has(arg));
+
+if (unsupportedArgs.length > 0) {
+  throw new Error(`Unsupported arguments: ${unsupportedArgs.join(", ")}`);
+}
+
+process.env.CURSORLESS_MODE = "test";
+
+if (args.has("--subset")) {
+  process.env.CURSORLESS_RUN_TEST_SUBSET = "true";
+}
+
+if (args.has("--update")) {
+  process.env.CURSORLESS_TEST_UPDATE_FIXTURES = "true";
+}
+
+void (async () => {
+  const { TestType, runAllTests } = await import("../runAllTests");
+
+  await runAllTests(TestType.unit);
+})();

packages/test-harness/src/scripts/runVscodeTestsCI.ts

@@ -6,6 +6,8 @@ import { getCursorlessRepoRoot } from "@cursorless/node-common";
 import * as path from "node:path";
 import { launchVscodeAndRunTests } from "../launchVscodeAndRunTests";
 
+process.env.CURSORLESS_MODE = "test";
+
 void (async () => {
   const extensionTestsPath = path.resolve(
     getCursorlessRepoRoot(),