Add basic semantic highlighting

src/indexing/DocumentIndexer.ts

@@ -12,6 +12,7 @@ const INDEXING_DELAY = 500 // Delay (in ms) after keystroke before attempting to
  */
 export default class DocumentIndexer {
     private readonly pendingFilesToIndex = new Map<string, NodeJS.Timeout>()
+    private onIndexed?: (uri: string) => void
 
     constructor (
         private readonly indexer: Indexer,
@@ -42,6 +43,7 @@ export default class DocumentIndexer {
      */
     indexDocument (textDocument: TextDocument): void {
         void this.indexer.indexDocument(textDocument)
+        this.onIndexed?.(textDocument.uri)
     }
 
     /**
@@ -73,5 +75,11 @@ export default class DocumentIndexer {
         if (!this.fileInfoIndex.codeInfoCache.has(uri)) {
             await this.indexer.indexDocument(textDocument)
         }
+
+        this.onIndexed?.(uri)
+    }
+
+    setOnIndexed (callback: (uri: string) => void): void {
+        this.onIndexed = callback
     }
 }

src/providers/semanticTokens/SemanticTokensProvider.ts

@@ -0,0 +1,112 @@
+import { SemanticTokens, SemanticTokensParams, TextDocuments, Range } from 'vscode-languageserver'
+import MatlabLifecycleManager from '../../lifecycle/MatlabLifecycleManager'
+import { TextDocument } from 'vscode-languageserver-textdocument'
+import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../indexing/FileInfoIndex'
+import DocumentIndexer from '../../indexing/DocumentIndexer'
+
+interface VariableToken {
+    range: Range
+    typeIndex: number
+}
+
+class SemanticTokensProvider {
+    constructor (
+        protected readonly matlabLifecycleManager: MatlabLifecycleManager,
+        protected readonly documentIndexer: DocumentIndexer,
+        protected readonly fileInfoIndex: FileInfoIndex
+    ) { }
+
+    async handleSemanticTokensRequest (
+        params: SemanticTokensParams,
+        documentManager: TextDocuments<TextDocument>
+    ): Promise<SemanticTokens | null> {
+        // This request will be called constantly, should not connect to MATLAB just because it was called
+        const matlabConnection = await this.matlabLifecycleManager.getMatlabConnection(false)
+        if (matlabConnection == null) {
+            // If MATLAB is not connected, fall back to textmate
+            return null
+        }
+
+        const textDocument = documentManager.get(params.textDocument.uri)
+        if (textDocument == null) return null
+
+        await this.documentIndexer.ensureDocumentIndexIsUpdated(textDocument)
+
+        const codeInfo = this.fileInfoIndex.codeInfoCache.get(params.textDocument.uri)
+        if (codeInfo == null) {
+            return { data: [] }
+        }
+
+        const tokens: VariableToken[] = []
+        this.collectVariableTokens(codeInfo.globalScopeInfo, tokens)
+
+        // Sort tokens by their position in the document (line and character)
+        // This is necessary to encode them using relative positions
+        tokens.sort((a, b) => {
+            const lineDiff = a.range.start.line - b.range.start.line
+            if (lineDiff !== 0) return lineDiff
+
+            return a.range.start.character - b.range.start.character
+        })
+
+        const data: number[] = []
+        let prevLine = 0
+        let prevStart = 0
+
+        // Encode semantic tokens using relative line and character positions
+        for (const token of tokens) {
+            const line = token.range.start.line
+            const start = token.range.start.character
+            const length = token.range.end.character - token.range.start.character
+
+            const deltaLine = line - prevLine
+            const deltaStart = deltaLine === 0 ? start - prevStart : start
+
+            data.push(deltaLine, deltaStart, length, token.typeIndex, 0)
+            prevLine = line
+            prevStart = start
+        }
+
+        return { data }
+    }
+
+    private collectVariableTokens (
+        scope: MatlabGlobalScopeInfo | MatlabFunctionScopeInfo,
+        tokens: VariableToken[]
+    ): void {
+        // Variables: highlight only the first component as variable
+        for (const item of scope.variables.values()) {
+            for (const ref of item.references) {
+                tokens.push({ range: ref.components[0].range, typeIndex: 1 }) // variable
+            }
+        }
+
+        // Functions/unbound: highlight only the first component as function
+        for (const item of scope.functionOrUnboundReferences.values()) {
+            for (const ref of item.references) {
+                tokens.push({ range: ref.components[0].range, typeIndex: 0 }) // function
+            }
+        }
+
+        // Class scope
+        const classScope = (scope as MatlabGlobalScopeInfo).classScope;
+        if (classScope != null) {
+            for (const nestedFunc of classScope.functionScopes.values()) {
+                if (nestedFunc.functionScopeInfo != null) {
+                    this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens);
+                }
+            }
+        }
+
+        // Function scopes
+        for (const nestedFunc of scope.functionScopes.values()) {
+            if (nestedFunc.functionScopeInfo != null) {
+                this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens)
+            }
+        }
+    }
+}
+
+export const SEMANTIC_TOKEN_TYPES = ['function', 'variable']
+export const SEMANTIC_TOKEN_MODIFIERS: string[] = []
+export default SemanticTokensProvider

src/server.ts

@@ -1,7 +1,7 @@
 // Copyright 2022 - 2025 The MathWorks, Inc.
 
 import { TextDocument } from 'vscode-languageserver-textdocument'
-import { ClientCapabilities, InitializeParams, InitializeResult, TextDocuments } from 'vscode-languageserver/node'
+import { ClientCapabilities, InitializeParams, InitializeResult, TextDocuments, SemanticTokensRequest, SemanticTokensParams } from 'vscode-languageserver/node'
 import DocumentIndexer from './indexing/DocumentIndexer'
 import WorkspaceIndexer from './indexing/WorkspaceIndexer'
 import ConfigurationManager, { ConnectionTiming } from './lifecycle/ConfigurationManager'
@@ -22,6 +22,7 @@ import PathResolver from './providers/navigation/PathResolver'
 import Indexer from './indexing/Indexer'
 import RenameSymbolProvider from './providers/rename/RenameSymbolProvider'
 import HighlightSymbolProvider from './providers/highlighting/HighlightSymbolProvider'
+import SemanticTokensProvider, { SEMANTIC_TOKEN_TYPES, SEMANTIC_TOKEN_MODIFIERS } from './providers/semanticTokens/SemanticTokensProvider'
 import { RequestType } from './indexing/SymbolSearchService'
 import { cacheAndClearProxyEnvironmentVariables } from './utils/ProxyUtils'
 import MatlabDebugAdaptorServer from './debug/MatlabDebugAdaptorServer'
@@ -73,6 +74,7 @@ export async function startServer (): Promise<void> {
     const navigationSupportProvider = new NavigationSupportProvider(matlabLifecycleManager, fileInfoIndex, indexer, documentIndexer, pathResolver)
     const renameSymbolProvider = new RenameSymbolProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex)
     const highlightSymbolProvider = new HighlightSymbolProvider(matlabLifecycleManager, documentIndexer, indexer, fileInfoIndex)
+    const semanticTokensProvider = new SemanticTokensProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex)
 
     let pathSynchronizer: PathSynchronizer | null
 
@@ -142,7 +144,14 @@ export async function startServer (): Promise<void> {
                 renameProvider: {
                     prepareProvider: true
                 },
-                documentHighlightProvider: true
+                documentHighlightProvider: true,
+                semanticTokensProvider: {
+                    legend: {
+                        tokenTypes: SEMANTIC_TOKEN_TYPES,
+                        tokenModifiers: SEMANTIC_TOKEN_MODIFIERS
+                    },
+                    full: true
+                }
             }
         }
 
@@ -264,7 +273,7 @@ export async function startServer (): Promise<void> {
         reportFileOpened(params.document)
         void lintingSupportProvider.lintDocument(params.document)
         void documentIndexer.indexDocument(params.document)
-        
+
         void navigationSupportProvider.handleDocumentSymbol(params.document.uri, documentManager, RequestType.DocumentSymbol)
     })
 
@@ -361,6 +370,28 @@ export async function startServer (): Promise<void> {
     connection.onDocumentHighlight(async params => {
         return await highlightSymbolProvider.handleDocumentHighlightRequest(params, documentManager)
     })
+
+    /** -------------- SEMANTIC TOKENS SUPPORT --------------- **/
+    connection.onRequest(SemanticTokensRequest.method, async (params: SemanticTokensParams) => {
+        return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager)
+    })
+
+    // Ensures that semantic tokens are refreshed after indexing,
+    // so highlighting is updated after opening the editor.
+    documentIndexer.setOnIndexed(() => {
+        scheduleSemanticRefresh()
+    })
+
+    let refreshTimer: NodeJS.Timeout | undefined
+
+    function scheduleSemanticRefresh (): void {
+        if (refreshTimer != null) clearTimeout(refreshTimer)
+
+        // Delay sending the refresh notification to batch multiple indexing updates together
+        refreshTimer = setTimeout(() => {
+            void connection.sendRequest('workspace/semanticTokens/refresh')
+        }, 150)
+    }
 }
 
 /** -------------------- Helper Functions -------------------- **/