fix: restore semantic syntax highlighting

Author: pallyoung

packages/core/src/domain/lsp.test.ts

@@ -8,6 +8,7 @@ import type {
   LspHoverResult,
   LspLocation,
   LspRuntimeMode,
+  LspSemanticTokens,
   LspSessionSummary,
   LspToolInstallFailure,
   LspToolInstallJobSnapshot,
@@ -83,6 +84,7 @@ describe("LSP shared surface", () => {
         references: boolean;
         hover: boolean;
         documentSymbols: boolean;
+        semanticTokens: boolean;
         diagnostics: boolean;
       };
     }>();
@@ -189,6 +191,11 @@ describe("LSP shared surface", () => {
       version?: number;
     }>();
 
+    expectTypeOf<LspSemanticTokens>().toEqualTypeOf<{
+      resultId?: string;
+      data: number[];
+    }>();
+
     type LspDiagnosticsUpdatedEvent = Extract<DomainEvent, { type: "lsp.diagnostics.updated" }>;
     expectTypeOf<LspDiagnosticsUpdatedEvent>().toMatchTypeOf<{
       type: "lsp.diagnostics.updated";

packages/core/src/domain/lsp.ts

@@ -2,6 +2,45 @@ export type LspServerKind = "typescript" | "python" | "go" | "rust" | "vue";
 export type LspToolSource = "override" | "managed" | "bundled" | "system";
 export type LspRuntimeMode = "auto" | "off";
 
+export const LSP_SEMANTIC_TOKEN_TYPES = [
+  "namespace",
+  "type",
+  "class",
+  "enum",
+  "interface",
+  "struct",
+  "typeParameter",
+  "parameter",
+  "variable",
+  "property",
+  "enumMember",
+  "event",
+  "function",
+  "method",
+  "macro",
+  "keyword",
+  "modifier",
+  "comment",
+  "string",
+  "number",
+  "regexp",
+  "operator",
+  "decorator",
+] as const;
+
+export const LSP_SEMANTIC_TOKEN_MODIFIERS = [
+  "declaration",
+  "definition",
+  "readonly",
+  "static",
+  "deprecated",
+  "abstract",
+  "async",
+  "modification",
+  "documentation",
+  "defaultLibrary",
+] as const;
+
 export interface LspRange {
   startLine: number;
   startColumn: number;
@@ -36,6 +75,11 @@ export interface LspDocumentSymbol {
   children?: LspDocumentSymbol[];
 }
 
+export interface LspSemanticTokens {
+  resultId?: string;
+  data: number[];
+}
+
 export interface LspSessionSummary {
   workspaceId: string;
   serverKind: LspServerKind;
@@ -47,6 +91,7 @@ export interface LspSessionSummary {
     references: boolean;
     hover: boolean;
     documentSymbols: boolean;
+    semanticTokens: boolean;
     diagnostics: boolean;
   };
 }

packages/server/src/__tests__/fixtures/fake-lsp-server.js

@@ -14,7 +14,12 @@ const connection = createMessageConnection(
 );
 
 const docs = new Map();
-const exitAfterInitMs = Number(process.env.CODER_STUDIO_FAKE_LSP_EXIT_AFTER_INIT_MS ?? "0");
+const exitAfterInitArg = process.argv.find((arg) => arg.startsWith("--exit-after-init-ms="));
+const exitAfterInitMs = Number(
+  exitAfterInitArg?.slice("--exit-after-init-ms=".length) ??
+    process.env.CODER_STUDIO_FAKE_LSP_EXIT_AFTER_INIT_MS ??
+    "0"
+);
 const hoverDelayMs = Number(process.env.CODER_STUDIO_FAKE_LSP_HOVER_DELAY_MS ?? "0");
 const initDelayMs = Number(process.env.CODER_STUDIO_FAKE_LSP_INIT_DELAY_MS ?? "0");
 const stderrOnInit = process.env.CODER_STUDIO_FAKE_LSP_STDERR_ON_INIT ?? "";
@@ -44,6 +49,13 @@ connection.onRequest("initialize", async () => {
       referencesProvider: true,
       hoverProvider: true,
       documentSymbolProvider: true,
+      semanticTokensProvider: {
+        legend: {
+          tokenTypes: ["function", "variable", "class", "typeAlias", "builtinType"],
+          tokenModifiers: ["declaration", "readonly"],
+        },
+        full: true,
+      },
       textDocumentSync: 1,
     },
   };
@@ -221,6 +233,20 @@ connection.onRequest("textDocument/documentSymbol", ({ textDocument }) => {
   ];
 });
 
+connection.onRequest("textDocument/semanticTokens/full", ({ textDocument }) => {
+  if (!textDocument.uri.endsWith("/shared.ts")) {
+    return { data: [] };
+  }
+
+  return {
+    resultId: "semantic-1",
+    data: [
+      // sharedValue: variable + declaration
+      0, 13, 11, 1, 1,
+    ],
+  };
+});
+
 function publishDiagnostics(uri) {
   const text = docs.get(uri) ?? readFileSync(new URL(uri), "utf8");
   const diagnostics = text.includes("missingSymbol")

packages/server/src/__tests__/lsp-commands.test.ts

@@ -38,6 +38,7 @@ class FakeLspManager {
           references: true,
           hover: true,
           documentSymbols: true,
+          semanticTokens: true,
           diagnostics: true,
         },
       },
@@ -79,6 +80,13 @@ class FakeLspManager {
   async documentSymbols() {
     return [];
   }
+
+  async semanticTokens() {
+    return {
+      resultId: "semantic-1",
+      data: [0, 13, 11, 8, 1],
+    };
+  }
 }
 
 class FakeLspToolInstallManager {
@@ -206,6 +214,27 @@ describe("LSP commands", () => {
         expect.objectContaining({ path: "e2e/fixtures/lsp-workspace/shared.ts" }),
       ])
     );
+
+    const semanticTokens = await dispatch(
+      {
+        kind: "command",
+        id: crypto.randomUUID(),
+        op: "lsp.semanticTokens",
+        args: {
+          workspaceId,
+          path: "e2e/fixtures/lsp-workspace/shared.ts",
+        },
+      },
+      ctx
+    );
+
+    expect(semanticTokens.ok).toBe(true);
+    expect(semanticTokens.data).toEqual(
+      expect.objectContaining({
+        resultId: "semantic-1",
+        data: [0, 13, 11, 8, 1],
+      })
+    );
   });
 
   it("exposes lsp runtime status and install commands", async () => {

packages/server/src/commands/lsp.ts

@@ -197,3 +197,12 @@ registerCommand(
   }),
   async (args, ctx) => ctx.lspMgr.documentSymbols(args)
 );
+
+registerCommand(
+  "lsp.semanticTokens",
+  z.object({
+    workspaceId: z.string(),
+    path: z.string(),
+  }),
+  async (args, ctx) => ctx.lspMgr.semanticTokens(args)
+);

packages/server/src/lsp/manager.ts

@@ -5,6 +5,7 @@ import type {
   LspHoverResult,
   LspLocation,
   LspRuntimeMode,
+  LspSemanticTokens,
   LspSessionSummary,
   Workspace,
 } from "@coder-studio/core";
@@ -37,6 +38,7 @@ interface LspSessionLike {
   references(input: { path: string; line: number; column: number }): Promise<LspLocation[] | null>;
   hover(input: { path: string; line: number; column: number }): Promise<LspHoverResult | null>;
   documentSymbols(input: { path: string }): Promise<LspDocumentSymbol[] | null>;
+  semanticTokens(input: { path: string }): Promise<LspSemanticTokens | null>;
 }
 
 interface ManagedSessionEntry {
@@ -340,6 +342,14 @@ export class LspManager {
     return session ? await session.documentSymbols(input) : null;
   }
 
+  async semanticTokens(input: { workspaceId: string; path: string }) {
+    if (this.runtimeMode === "off") {
+      return null;
+    }
+    const session = await this.getSessionForPath(input.workspaceId, input.path);
+    return session ? await session.semanticTokens(input) : null;
+  }
+
   async disposeWorkspace(workspaceId: string): Promise<void> {
     const keys = Array.from(this.sessions.keys()).filter((key) =>
       key.startsWith(`${workspaceId}::`)

packages/server/src/lsp/session.test.ts

@@ -1,4 +1,5 @@
 import { join } from "node:path";
+import { LSP_SEMANTIC_TOKEN_MODIFIERS, LSP_SEMANTIC_TOKEN_TYPES } from "@coder-studio/core";
 import { describe, expect, it, vi } from "vitest";
 import { LspSession } from "./session.js";
 
@@ -53,7 +54,8 @@ describe.sequential("LspSession", () => {
       },
     });
 
-    await session.start();
+    const summary = await session.start();
+    expect(summary.capabilities.semanticTokens).toBe(true);
     await session.openDocument({
       path: "e2e/fixtures/lsp-workspace/broken.ts",
       languageId: "typescript",
@@ -102,6 +104,21 @@ describe.sequential("LspSession", () => {
 
     expect(symbols?.[0]?.name).toBe("sharedValue");
 
+    const semanticTokens = await session.semanticTokens({
+      path: "e2e/fixtures/lsp-workspace/shared.ts",
+    });
+
+    expect(semanticTokens).toEqual({
+      resultId: "semantic-1",
+      data: [
+        0,
+        13,
+        11,
+        LSP_SEMANTIC_TOKEN_TYPES.indexOf("variable"),
+        1 << LSP_SEMANTIC_TOKEN_MODIFIERS.indexOf("declaration"),
+      ],
+    });
+
     expect(diagnostics).toHaveBeenCalledWith(
       expect.objectContaining({
         workspaceId: "ws-1",
@@ -493,66 +510,55 @@ describe.sequential("LspSession", () => {
   it("kills the companion process when the primary exits", async () => {
     // If Volar crashes we must not leave the TypeScript companion alive
     // (otherwise idle-TTL cleanup leaks a process per session).
-    const previous = process.env.CODER_STUDIO_FAKE_LSP_EXIT_AFTER_INIT_MS;
-    process.env.CODER_STUDIO_FAKE_LSP_EXIT_AFTER_INIT_MS = "150";
-
-    try {
-      const session = new LspSession({
-        workspaceId: "ws-1",
-        workspacePath: process.cwd(),
-        spec: {
-          serverKind: "vue",
-          // Primary exits 150ms after initialize.
+    const session = new LspSession({
+      workspaceId: "ws-1",
+      workspacePath: process.cwd(),
+      spec: {
+        serverKind: "vue",
+        // Primary exits 150ms after initialize.
+        command: "node",
+        args: [FAKE_LSP, "--exit-after-init-ms=150"],
+        rootPath: process.cwd(),
+        companion: {
+          // Companion stays alive normally.
           command: "node",
           args: [FAKE_LSP],
-          rootPath: process.cwd(),
-          companion: {
-            // Companion stays alive normally.
-            command: "node",
-            args: [FAKE_LSP],
-          },
-          bridges: { tsserverRequest: true },
-        },
-        onDiagnostics: vi.fn(),
-        requestTimeoutMs: 2000,
-        logger: {
-          info: vi.fn(),
-          warn: vi.fn(),
-          error: vi.fn(),
         },
-      });
+        bridges: { tsserverRequest: true },
+      },
+      onDiagnostics: vi.fn(),
+      requestTimeoutMs: 2000,
+      logger: {
+        info: vi.fn(),
+        warn: vi.fn(),
+        error: vi.fn(),
+      },
+    });
 
-      // Pull the companion field via a typed accessor for inspection.
-      type WithCompanion = LspSession & {
-        companion: null | { child: { killed: boolean } };
-      };
+    // Pull the companion field via a typed accessor for inspection.
+    type WithCompanion = LspSession & {
+      companion: null | { child: { killed: boolean } };
+    };
 
-      await session.start();
-      // Companion was spawned alongside primary.
-      expect((session as WithCompanion).companion).not.toBeNull();
-      const companionChild = (session as WithCompanion).companion?.child;
-      expect(companionChild).toBeDefined();
-
-      // Wait long enough for the primary to exit and the termination handler
-      // to fire.
-      await vi.waitFor(
-        () => {
-          expect((session as WithCompanion).companion).toBeNull();
-        },
-        { timeout: 2000 }
-      );
-      // The companion's process should have received SIGTERM.
-      expect(companionChild?.killed).toBe(true);
-      expect(session.getSummary().status).toBe("stopped");
+    await session.start();
+    // Companion was spawned alongside primary.
+    expect((session as WithCompanion).companion).not.toBeNull();
+    const companionChild = (session as WithCompanion).companion?.child;
+    expect(companionChild).toBeDefined();
 
-      await session.stop();
-    } finally {
-      if (previous === undefined) {
-        delete process.env.CODER_STUDIO_FAKE_LSP_EXIT_AFTER_INIT_MS;
-      } else {
-        process.env.CODER_STUDIO_FAKE_LSP_EXIT_AFTER_INIT_MS = previous;
-      }
-    }
+    // Wait long enough for the primary to exit and the termination handler
+    // to fire.
+    await vi.waitFor(
+      () => {
+        expect((session as WithCompanion).companion).toBeNull();
+      },
+      { timeout: 2000 }
+    );
+    // The companion's process should have received SIGTERM.
+    expect(companionChild?.killed).toBe(true);
+    expect(session.getSummary().status).toBe("stopped");
+
+    await session.stop();
   });
 
   it("stops the companion when the session is explicitly stopped", async () => {