Merge remote-tracking branch 'origin/main' into feature/1010-pom-leading-for-dsls/final

Author: toinehartman

.github/workflows/build.yaml

@@ -43,8 +43,8 @@ jobs:
         working-directory: ./rascal-vscode-extension
         run: |
           npm ci
-          npm run compile-tests
-          npm run normalTest
+          npm run compile:tests
+          npm run test:normal
       
   ui-test:
     strategy:
@@ -80,7 +80,7 @@ jobs:
         working-directory: ./rascal-vscode-extension
         run: |
           npm ci
-          npm run compile-tests
+          npm run compile:tests
 
       - name: Cache vscode downloads 
         id: cache-vscode
@@ -182,15 +182,16 @@ jobs:
             sed -i "1i⚠️ This is pre-release build, based on [CI build $GITHUB_RUN_NUMBER](https://github.com/usethesource/rascal-language-servers/actions/runs/$GITHUB_RUN_ID). ⚠️\n\n" README.md
           sed -i "s/\\(\"version\":.*\\)\\-head/\\1-head$GITHUB_RUN_NUMBER/" package.json
 
-      - name: Package & compile extension 
+      - name: Compile extension 
         working-directory: ./rascal-vscode-extension
         run: |
           npm ci
-          npm run license-check
-          npm run esbuild
+          npm run license:check
           npm run lint 
+          npm run check:types
+          npm run compile
 
-      - name: package extension
+      - name: Package extension
         working-directory: rascal-vscode-extension
         run: |
           npm run lsp4j:package

rascal-lsp/src/main/checkerframework/lsp4j.astub

@@ -451,3 +451,20 @@ public class SemanticTokensCapabilities extends DynamicRegistrationCapabilities
   public @Nullable Boolean getServerCancelSupport() {}
   public @Nullable Boolean getAugmentsSyntaxTokens() {}
 }
+
+package org.eclipse.lsp4j;
+
+import org.checkerframework.checker.nullness.qual.*;
+
+public class DocumentFilter {
+  public DocumentFilter(@Nullable String language, @Nullable String scheme, @Nullable Either<String, RelativePattern> pattern) {}
+}
+
+package org.eclipse.lsp4j;
+
+import org.checkerframework.checker.nullness.qual.*;
+
+public class Registration {
+  public Registration(String id, String method) {}
+  public Registration(String id, String method, @Nullable Object registerOptions) {}
+}

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/TextDocumentState.java

@@ -38,6 +38,7 @@
 import org.apache.logging.log4j.Logger;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.rascalmpl.library.util.ParseErrorRecovery;
+import org.rascalmpl.uri.FileAttributes;
 import org.rascalmpl.values.IRascalValueFactory;
 import org.rascalmpl.values.parsetrees.ITree;
 import org.rascalmpl.vscode.lsp.parametric.NoContributions.NoContributionException;
@@ -64,6 +65,7 @@ public class TextDocumentState {
     private final BiFunction<ISourceLocation, String, CompletableFuture<ITree>> parser;
     private final ISourceLocation location;
     private final ExecutorService exec;
+    private final FileAttributes attributesOnDisk;
 
     private final AtomicReference<Versioned<Update>> current;
     private final AtomicReference<@Nullable Versioned<ITree>> lastWithoutErrors;
@@ -73,10 +75,11 @@ public TextDocumentState(
             BiFunction<ISourceLocation, String, CompletableFuture<ITree>> parser,
             ISourceLocation location,
             int initialVersion, String initialContent, long initialTimestamp,
-            ExecutorService exec) {
+            ExecutorService exec, FileAttributes attributesOnDisk) {
 
         this.parser = parser;
         this.location = location;
+        this.attributesOnDisk = attributesOnDisk;
         this.lastWithoutErrors = new AtomicReference<>();
         this.last = new AtomicReference<>();
         this.exec = exec;
@@ -89,6 +92,13 @@ public ISourceLocation getLocation() {
         return location;
     }
 
+    /**
+     * The file attributes (aka stat) of the location on disk, before it was opened by VS Code
+     */
+    public FileAttributes getAttributesOnDisk() {
+        return attributesOnDisk;
+    }
+
     public CompletableFuture<Versioned<List<Diagnostics.Template>>> update(int version, String content, long timestamp) {
         var u = new Update(version, content, timestamp);
         Versioned.replaceIfNewer(current, new Versioned<>(version, u));
@@ -266,6 +276,6 @@ public long getLastModified() {
 
     public TextDocumentState changeParser(BiFunction<ISourceLocation, String, CompletableFuture<ITree>> parsing) {
         var c = getCurrentContent();
-        return new TextDocumentState(parsing, this.location, c.version(), c.get(), getLastModified(), exec);
+        return new TextDocumentState(parsing, this.location, c.version(), c.get(), getLastModified(), exec, getAttributesOnDisk());
     }
 }

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/TextDocumentStateManager.java

@@ -49,6 +49,7 @@
 import org.eclipse.lsp4j.jsonrpc.ResponseErrorException;
 import org.eclipse.lsp4j.jsonrpc.messages.ResponseError;
 import org.eclipse.lsp4j.jsonrpc.messages.ResponseErrorCode;
+import org.rascalmpl.uri.FileAttributes;
 import org.rascalmpl.uri.URIResolverRegistry;
 import org.rascalmpl.util.locations.ColumnMaps;
 import org.rascalmpl.util.locations.LineColumnOffsetMap;
@@ -58,7 +59,6 @@
 import org.rascalmpl.vscode.lsp.util.Lists;
 import org.rascalmpl.vscode.lsp.util.Versioned;
 import org.rascalmpl.vscode.lsp.util.locations.Locations;
-
 import io.usethesource.vallang.ISourceLocation;
 
 /**
@@ -153,9 +153,25 @@ protected TextDocumentState getFile(ISourceLocation loc) {
         }
     }
 
+    private FileAttributes safeStat(ISourceLocation loc, long timestamp) {
+        try {
+            return URIResolverRegistry.getInstance().stat(loc);
+        } catch (IOException e) {
+            // generate a fallback
+            // for example for untitled files, text document content provided,
+            // or other files that are not accessible to our VFS
+            return new FileAttributes(
+                true, true,
+                timestamp, timestamp,
+                true, false,
+                0
+            );
+        }
+
+    }
+
     protected TextDocumentState openFile(TextDocumentItem doc, Function<ISourceLocation, BiFunction<ISourceLocation, String, CompletableFuture<ITree>>> parserGetter, long timestamp, ExecutorService exec)  {
-        return files.computeIfAbsent(Locations.toLoc(doc),
-            l -> new TextDocumentState(parserGetter.apply(l), l, doc.getVersion(), doc.getText(), timestamp, exec));
+        return files.computeIfAbsent(Locations.toLoc(doc), l -> new TextDocumentState(parserGetter.apply(l), l, doc.getVersion(), doc.getText(), timestamp, exec, safeStat(l, timestamp)));
     }
 
     private void invalidateColumnMaps(ISourceLocation loc) {
@@ -191,12 +207,13 @@ protected void closeFile(ISourceLocation loc) {
         return files.keySet();
     }
 
-    protected void updateContents(DidChangeTextDocumentParams change, long timestamp) {
+    protected TextDocumentState updateContents(DidChangeTextDocumentParams change, long timestamp) {
         var doc = change.getTextDocument();
         logger.trace("New contents for {}", doc);
         TextDocumentState file = getFile(Locations.toLoc(doc));
         invalidateColumnMaps(file.getLocation());
         handleParsingErrors(file, file.update(doc.getVersion(), Lists.last(change.getContentChanges()).getText(), timestamp));
+        return file;
     }
 
     protected void handleParsingErrors(TextDocumentState file, CompletableFuture<Versioned<List<Diagnostics.Template>>> diagnosticsAsync) {

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java

@@ -26,6 +26,7 @@
  */
 package org.rascalmpl.vscode.lsp.parametric;
 
+import java.net.URI;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -39,12 +40,14 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.ExecutorService;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -112,7 +115,6 @@
 import org.eclipse.lsp4j.TextDocumentIdentifier;
 import org.eclipse.lsp4j.TextDocumentItem;
 import org.eclipse.lsp4j.TextDocumentSyncKind;
-import org.eclipse.lsp4j.VersionedTextDocumentIdentifier;
 import org.eclipse.lsp4j.WorkspaceEdit;
 import org.eclipse.lsp4j.WorkspaceFolder;
 import org.eclipse.lsp4j.jsonrpc.ResponseErrorException;
@@ -138,6 +140,7 @@
 import org.rascalmpl.vscode.lsp.parametric.capabilities.CompletionCapability;
 import org.rascalmpl.vscode.lsp.parametric.capabilities.FileOperationCapability;
 import org.rascalmpl.vscode.lsp.parametric.capabilities.ICapabilityParams;
+import org.rascalmpl.vscode.lsp.parametric.capabilities.SemanticTokensCapability;
 import org.rascalmpl.vscode.lsp.parametric.model.ParametricFileFacts;
 import org.rascalmpl.vscode.lsp.parametric.model.ParametricSummary;
 import org.rascalmpl.vscode.lsp.parametric.model.ParametricSummary.SummaryLookup;
@@ -180,6 +183,7 @@ public class ParametricTextDocumentService extends TextDocumentStateManager impl
 
     private final String dedicatedLanguageName;
     private final SemanticTokenizer tokenizer = new SemanticTokenizer();
+    private final Set<String> extensionLessSchemes = new CopyOnWriteArraySet<>();
     private final boolean exitWhenEmpty;
 
     private @MonotonicNonNull LanguageClient client;
@@ -228,6 +232,7 @@ public void initializeServerCapabilities(ClientCapabilities clientCapabilities,
         // Since the initialize request is the very first request after connecting, we can initialize the capabilities here
         // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#initialize
         dynamicCapabilities = new CapabilityRegistration(availableClient(), exec, clientCapabilities
+            , new SemanticTokensCapability()
             , new CompletionCapability()
             , /* new FileOperationCapability.DidCreateFiles(exec), */ new FileOperationCapability.DidRenameFiles(exec), new FileOperationCapability.DidDeleteFiles(exec)
         );
@@ -246,7 +251,6 @@ private static void setStaticServerCapabilities(String dedicatedLanguageName, Se
         result.setReferencesProvider(true);
         result.setDocumentSymbolProvider(true);
         result.setImplementationProvider(true);
-        result.setSemanticTokensProvider(SemanticTokenizer.options());
         result.setCodeActionProvider(true);
         result.setCodeLensProvider(new CodeLensOptions(false));
         result.setRenameProvider(new RenameOptions(true));
@@ -313,15 +317,31 @@ public void didOpen(DidOpenTextDocumentParams params) {
         logger.debug("Did Open file: {}", params.getTextDocument());
         TextDocumentState file = open(params.getTextDocument(), timestamp);
         handleParsingErrors(file, file.getCurrentDiagnosticsAsync());
-        triggerAnalyzer(params.getTextDocument(), NORMAL_DEBOUNCE);
+        triggerAnalyzer(file, NORMAL_DEBOUNCE);
+
+        // Discover capabilities
+        discoverExtensionLessScheme(URIUtil.assumeCorrect(params.getTextDocument().getUri()));
+    }
+
+    /**
+     * Captures extensions-less schemes and updates dynamic capabilities to apply to them.
+     * @param uri A URI that should be associated with the parametric server.
+     */
+    private void discoverExtensionLessScheme(URI uri) {
+        var ext = URIUtil.getExtension(Locations.toLoc(uri));
+        // We want the original scheme, not the one possibly modified when converting URI -> loc
+        if (ext.equals("") && extensionLessSchemes.add(uri.getScheme())) {
+            // Should be called from the main, single-threaded request pool
+            updateCapabilities();
+        }
     }
 
     @Override
     public void didChange(DidChangeTextDocumentParams params) {
         var timestamp = System.currentTimeMillis();
         logger.debug("Did Change file: {}", params.getTextDocument().getUri());
-        updateContents(params, timestamp);
-        triggerAnalyzer(params.getTextDocument(), NORMAL_DEBOUNCE);
+        var state = updateContents(params, timestamp);
+        triggerAnalyzer(state, NORMAL_DEBOUNCE);
     }
 
     @Override
@@ -364,21 +384,13 @@ public void didDeleteFiles(DeleteFilesParams params) {
         }
     }
 
-    private void triggerAnalyzer(TextDocumentItem doc, Duration delay) {
-        triggerAnalyzer(new VersionedTextDocumentIdentifier(doc.getUri(), doc.getVersion()), delay);
-    }
-
-    private void triggerAnalyzer(VersionedTextDocumentIdentifier doc, Duration delay) {
-        var location = Locations.toLoc(doc);
-        triggerAnalyzer(location, doc.getVersion(), delay);
-    }
-
-    private void triggerAnalyzer(ISourceLocation location, int version, Duration delay) {
-        if (safeLanguage(location).isPresent()) {
+    private void triggerAnalyzer(TextDocumentState state, Duration delay) {
+        var location = state.getLocation();
+        if (safeLanguage(state.getLocation()).isPresent()) {
             logger.trace("Triggering analyzer for {}", location);
             var fileFacts = facts(location);
             fileFacts.invalidateAnalyzer(location);
-            fileFacts.calculateAnalyzer(location, getFile(location).getCurrentTreeAsync(true), version, delay);
+            fileFacts.calculateAnalyzer(location, state.getCurrentTreeAsync(true), state.getCurrentContent(), delay);
         } else {
             logger.debug("Not triggering analyzer, since no language is registered for {}", location);
         }
@@ -941,11 +953,6 @@ public synchronized void registerLanguage(LanguageParameter lang) {
             this.registeredExtensions.put(extension, lang.getName());
         }
 
-        // `CapabilityRegistration::update` should never be called asynchronously, since that might re-order incoming updates.
-        // Since `registerLanguage` is called from a single-threaded pool, calling it here is safe.
-        // Note: `CapabilityRegistration::update` returns a void future, which we do not have to wait on.
-        availableCapabilities().update(buildLanguageParams());
-
         // If we opened any files with this extension before, now associate them with contributions
         var extensions = Arrays.asList(lang.getExtensions());
         for (var f : getOpenFiles()) {
@@ -954,6 +961,32 @@ public synchronized void registerLanguage(LanguageParameter lang) {
                 refreshFileState(f);
             }
         }
+
+        // Do this last, after the parser for open editors has been updated,
+        // since this might trigger new requests on the editor contents.
+        // `updateCapabilities`/`CapabilityRegistration::update` should never be called asynchronously, since that might re-order incoming updates.
+        // Since `registerLanguage` is called from a single-threaded pool, calling it here is safe.
+        // Note: `updateCapabilities` returns a void future, which we do not have to wait on.
+        updateCapabilities();
+    }
+
+    private CompletableFuture<Void> refreshEditors() {
+        var client = availableClient();
+        // Do all in parallel, and return a future that completes when each of these completes
+        return CompletableFutureUtils.reduce(List.of(
+            client.refreshCodeLenses(),
+            client.refreshDiagnostics(),
+            client.refreshFoldingRanges(),
+            client.refreshInlayHints(),
+            client.refreshInlineValues(),
+            client.refreshSemanticTokens()
+        ), (v1, v2) -> null);
+    }
+
+    private synchronized CompletableFuture<Void> updateCapabilities() {
+        return availableCapabilities()
+            .update(buildLanguageParams())
+            .thenCompose(v -> refreshEditors());
     }
 
     /**
@@ -972,6 +1005,11 @@ public ILanguageContributions contributions() {
             public Set<String> fileExtensions() {
                 return extensionsByLang.getOrDefault(e.getKey(), Collections.emptySet());
             }
+
+            @Override
+            public Set<String> extensionLessSchemes() {
+                return extensionLessSchemes;
+            }
         }).collect(Collectors.toSet());
     }
 
@@ -986,7 +1024,7 @@ private void refreshFileState(ISourceLocation f) {
         }
         // Update open editor
         handleParsingErrors(state, state.getCurrentDiagnosticsAsync());
-        triggerAnalyzer(f, state.getCurrentContent().version(), NORMAL_DEBOUNCE);
+        triggerAnalyzer(state, NORMAL_DEBOUNCE);
     }
 
     private static String buildContributionKey(LanguageParameter lang) {
@@ -1026,7 +1064,8 @@ public synchronized void unregisterLanguage(LanguageParameter lang) {
             System.exit(0);
         }
 
-        availableCapabilities().update(buildLanguageParams());
+        // Should be called from the main, single-threaded request pool
+        updateCapabilities();
     }
 
     @Override