Run scope tests as unit test outside of vscode (#3210)

We're getting quite a lot of scope test by now and running all of them
inside of the yes code is very slow. This updates allow us to run the
scope test as unit tests outside of the vscode. It also refactors the
test workflow so all unit tests run outside of the editors.

As a side defect of running into quite a lot of bugs for this
implementation I added better error messages, missing error logging,
missing types etc in various files.

Author: AndreasArvidsson

.github/workflows/test.yml

@@ -62,14 +62,25 @@ jobs:
       - name: Build
         run: pnpm --color --filter '!cursorless-org' --filter '!cursorless-org-*' build
 
-      - name: Run tests (Linux)
+      - name: Lint
+        run: pnpm --color lint
+
+      - name: Run unit tests (Linux)
         run: xvfb-run -a pnpm --color test
         if: runner.os == 'Linux'
 
-      - name: Run tests (Win,Mac)
+      - name: Run unit tests (Win,Mac)
         run: pnpm --color test
         if: runner.os != 'Linux'
 
+      - name: Run VSCode tests (Linux)
+        run: xvfb-run -a pnpm -F @cursorless/test-harness test:vscode
+        if: runner.os == 'Linux'
+
+      - name: Run VSCode tests (Win,Mac)
+        run: pnpm -F @cursorless/test-harness test:vscode
+        if: runner.os != 'Linux'
+
       - name: Run Talon-JS tests (Linux)
         run: xvfb-run -a pnpm -F @cursorless/test-harness test:talonJs
         if: runner.os == 'Linux' && matrix.app_version == 'stable'

.vscode/launch.json

@@ -202,41 +202,6 @@
       ]
     },
 
-    // Unit tests launch configs
-    {
-      "name": "Unit tests: Test",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceFolder}/packages/test-harness/dist/runUnitTestsOnly.cjs",
-      "env": {
-        "CURSORLESS_MODE": "test",
-        "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
-      },
-      "outFiles": ["${workspaceFolder}/**/out/**/*.js"],
-      "preLaunchTask": "VSCode: Build extension and tests",
-      "resolveSourceMapLocations": [
-        "${workspaceFolder}/**",
-        "!**/node_modules/**"
-      ]
-    },
-    {
-      "name": "Unit tests: Update test fixtures",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceFolder}/packages/test-harness/dist/runUnitTestsOnly.cjs",
-      "env": {
-        "CURSORLESS_MODE": "test",
-        "CURSORLESS_TEST_UPDATE_FIXTURES": "true",
-        "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
-      },
-      "outFiles": ["${workspaceFolder}/**/out/**/*.js"],
-      "preLaunchTask": "VSCode: Build extension and tests",
-      "resolveSourceMapLocations": [
-        "${workspaceFolder}/**",
-        "!**/node_modules/**"
-      ]
-    },
-
     // Docusaurus launch configs
     {
       "name": "Docusaurus: Run",

package.json

@@ -26,7 +26,7 @@
     "meta-updater:base": "pnpm --filter=@cursorless/meta-updater build && meta-updater",
     "preinstall": "npx only-allow pnpm",
     "test-compile": "tsc --build",
-    "test": "pnpm compile && pnpm lint && pnpm -F '!test-harness' test && pnpm -F test-harness test",
+    "test": "pnpm -r test",
     "generate-grammar": "pnpm -r generate-grammar",
     "transform-recorded-tests": "./packages/common/scripts/my-ts-node.js packages/cursorless-engine/src/scripts/transformRecordedTests/index.ts",
     "watch": "pnpm run -w --parallel '/^watch:.*/'",

packages/common/package.json

@@ -38,6 +38,7 @@
     "cross-spawn": "^7.0.6",
     "fast-check": "^4.5.3",
     "js-yaml": "^4.1.1",
-    "mocha": "^11.7.5"
+    "mocha": "^11.7.5",
+    "web-tree-sitter": "^0.26.6"
   }
 }

packages/common/scripts/my-ts-node.js

@@ -88,6 +88,7 @@ async function main() {
     bundle: true,
     format: "cjs",
     outfile: outFile,
+    external: ["./reporters/parallel-buffered", "./worker.js"],
   });
 
   const nodeProcess = runCommand(

packages/common/src/ide/fake/FakeIDE.ts

@@ -1,5 +1,10 @@
 import { pull } from "lodash-es";
-import type { EditableTextEditor, NotebookEditor, TextEditor } from "../..";
+import type {
+  EditableTextEditor,
+  Messages,
+  NotebookEditor,
+  TextEditor,
+} from "../..";
 import type { GeneralizedRange } from "../../types/GeneralizedRange";
 import type { TextDocument } from "../../types/TextDocument";
 import type { TextDocumentChangeEvent } from "../types/Events";
@@ -24,11 +29,11 @@ import FakeKeyValueStore from "./FakeKeyValueStore";
 import FakeMessages from "./FakeMessages";
 
 export class FakeIDE implements IDE {
-  configuration: FakeConfiguration = new FakeConfiguration();
-  messages: FakeMessages = new FakeMessages();
-  keyValueStore: FakeKeyValueStore = new FakeKeyValueStore();
-  clipboard: FakeClipboard = new FakeClipboard();
-  capabilities: FakeCapabilities = new FakeCapabilities();
+  configuration = new FakeConfiguration();
+  keyValueStore = new FakeKeyValueStore();
+  clipboard = new FakeClipboard();
+  capabilities = new FakeCapabilities();
+  messages: Messages;
 
   runMode: RunMode = "test";
   cursorlessVersion: string = "0.0.0";
@@ -37,6 +42,10 @@ export class FakeIDE implements IDE {
   private assetsRoot_: string | undefined;
   private quickPickReturnValue: string | undefined = undefined;
 
+  constructor(messages: Messages = new FakeMessages()) {
+    this.messages = messages;
+  }
+
   async flashRanges(_flashDescriptors: FlashDescriptor[]): Promise<void> {
     // empty
   }
@@ -57,8 +66,9 @@ export class FakeIDE implements IDE {
     dummyEvent;
   onDidChangeTextEditorVisibleRanges: Event<TextEditorVisibleRangesChangeEvent> =
     dummyEvent;
+  onDidChangeTextDocument: Event<TextDocumentChangeEvent> = dummyEvent;
 
-  public mockAssetsRoot(_assetsRoot: string) {
+  mockAssetsRoot(_assetsRoot: string) {
     this.assetsRoot_ = _assetsRoot;
   }
 
@@ -71,41 +81,41 @@ export class FakeIDE implements IDE {
   }
 
   get activeTextEditor(): TextEditor | undefined {
-    throw Error("Not implemented");
+    throw Error("activeTextEditor: not implemented");
   }
 
   get activeEditableTextEditor(): EditableTextEditor | undefined {
-    throw Error("Not implemented");
+    throw Error("activeEditableTextEditor: not implemented");
   }
 
   get visibleTextEditors(): TextEditor[] {
-    throw Error("Not implemented");
+    return [];
   }
 
   get visibleNotebookEditors(): NotebookEditor[] {
-    throw Error("Not implemented");
+    return [];
   }
 
   public getEditableTextEditor(_editor: TextEditor): EditableTextEditor {
-    throw Error("Not implemented");
+    throw Error("getEditableTextEditor: not implemented");
   }
 
   public findInDocument(_query: string, _editor: TextEditor): Promise<void> {
-    throw Error("Not implemented");
+    throw Error("findInDocument: not implemented");
   }
 
   public findInWorkspace(_query: string): Promise<void> {
-    throw Error("Not implemented");
+    throw Error("findInWorkspace: not implemented");
   }
 
   public openTextDocument(_path: string): Promise<TextEditor> {
-    throw Error("Not implemented");
+    throw Error("openTextDocument: not implemented");
   }
 
   public openUntitledTextDocument(
     _options: OpenUntitledTextDocumentOptions,
   ): Promise<TextEditor> {
-    throw Error("Not implemented");
+    throw Error("openUntitledTextDocument: not implemented");
   }
 
   public setQuickPickReturnValue(value: string | undefined) {
@@ -120,17 +130,11 @@ export class FakeIDE implements IDE {
   }
 
   public showInputBox(_options?: any): Promise<string | undefined> {
-    throw Error("Not implemented");
+    throw Error("showInputBox: not implemented");
   }
 
   executeCommand<T>(_command: string, ..._args: any[]): Promise<T | undefined> {
-    throw new Error("Method not implemented.");
-  }
-
-  public onDidChangeTextDocument(
-    _listener: (event: TextDocumentChangeEvent) => void,
-  ): Disposable {
-    throw Error("Not implemented");
+    throw new Error("executeCommand: not implemented");
   }
 
   disposeOnExit(...disposables: Disposable[]): () => void {

packages/common/src/index.ts

@@ -104,6 +104,7 @@ export * from "./util/DefaultMap";
 export * from "./util/disposableFrom";
 export * from "./util/ensureCommandShape";
 export * from "./util/getEnvironmentVariableStrict";
+export * from "./util/getErrorMessage";
 export * from "./util/itertools";
 export * from "./util/Notifier";
 export * from "./util/object";

packages/common/src/util/getErrorMessage.ts

@@ -0,0 +1,3 @@
+export function getErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

packages/cursorless-engine/package.json

@@ -41,14 +41,18 @@
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@types/chai": "^5.2.3",
     "@types/js-yaml": "^4.0.9",
     "@types/lodash-es": "^4.17.12",
     "@types/mocha": "^10.0.10",
     "@types/moo": "^0.5.10",
     "@types/nearley": "^2.11.5",
     "@types/sinon": "^21.0.0",
+    "chai": "^6.2.2",
     "js-yaml": "^4.1.1",
     "mocha": "^11.7.5",
-    "sinon": "^21.0.2"
+    "sinon": "^21.0.2",
+    "vscode-uri": "^3.1.0",
+    "web-tree-sitter": "^0.26.6"
   }
 }

packages/cursorless-engine/src/api/CursorlessEngineApi.ts

@@ -13,9 +13,11 @@ import type {
 } from "@cursorless/common";
 import type { CommandRunner } from "../CommandRunner";
 import type { StoredTargetMap } from "../core/StoredTargets";
+import type { LanguageDefinitions } from "../languages/LanguageDefinitions";
 
 export interface CursorlessEngine {
   commandApi: CommandApi;
+  languageDefinitions: LanguageDefinitions;
   scopeProvider: ScopeProvider;
   customSpokenFormGenerator: CustomSpokenFormGenerator;
   storedTargets: StoredTargetMap;