Merge pull request #424 from 1c-syntax/copilot/fix-scanner-parsing-error

Fix BSLHighlighter crash on multiline SDBL tokens (GROUP BY on separate lines)

Author: nixel2007

build.gradle.kts

@@ -42,6 +42,7 @@ dependencies {
     implementation("org.commonmark", "commonmark-ext-heading-anchor", commonmarkVersion)
 
     testImplementation("org.junit.jupiter", "junit-jupiter-api", "6.0.3")
+    testImplementation("org.junit.jupiter", "junit-jupiter-params", "6.0.3")
     testImplementation("org.assertj", "assertj-core", "3.27.7")
     testImplementation("org.mockito", "mockito-core", "5.21.0")
     testImplementation("org.sonarsource.sonarqube", "sonar-testing-harness", sonarQubeVersion) {

src/main/java/com/github/_1c_syntax/bsl/sonar/BSLHighlighter.java

@@ -28,11 +28,13 @@
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.Tokenizer;
 import org.eclipse.lsp4j.Range;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.sensor.SensorContext;
+import org.sonar.api.batch.sensor.highlighting.NewHighlighting;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 
 import javax.annotation.Nullable;
@@ -45,6 +47,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+@Slf4j
 @RequiredArgsConstructor
 public class BSLHighlighter {
 
@@ -158,19 +161,29 @@ public void saveHighlighting(InputFile inputFile, DocumentContext documentContex
 
     highlightingData.stream()
       .filter(HighlightingData::isActive)
-      .forEach(data ->
-        highlighting.highlight(
-          data.getRange().getStart().getLine(),
-          data.getRange().getStart().getCharacter(),
-          data.getRange().getEnd().getLine(),
-          data.getRange().getEnd().getCharacter(),
-          data.getType()
-        )
-      );
+      .forEach(data -> applyHighlighting(highlighting, data, inputFile));
 
     highlighting.save();
   }
 
+  private static void applyHighlighting(
+    NewHighlighting highlighting,
+    HighlightingData data,
+    InputFile inputFile
+  ) {
+    try {
+      highlighting.highlight(
+        data.getRange().getStart().getLine(),
+        data.getRange().getStart().getCharacter(),
+        data.getRange().getEnd().getLine(),
+        data.getRange().getEnd().getCharacter(),
+        data.getType()
+      );
+    } catch (IllegalArgumentException e) {
+      LOGGER.error("Unable to highlight file {}", inputFile, e);
+    }
+  }
+
   public void highlightToken(
     Token token,
     Collection<HighlightingData> highlightingData,
@@ -184,6 +197,11 @@ public void highlightToken(
     var charPositionInLine = token.getCharPositionInLine();
     var tokenText = token.getText().stripTrailing();
 
+    var newlineIndex = tokenText.indexOf('\n');
+    if (newlineIndex >= 0) {
+      tokenText = tokenText.substring(0, newlineIndex).stripTrailing();
+    }
+
     var range = Ranges.create(
       line,
       charPositionInLine,

src/test/java/com/github/_1c_syntax/bsl/sonar/BSLHighlighterTest.java

@@ -30,6 +30,8 @@
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.Vocabulary;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
 import org.sonar.api.batch.sensor.internal.SensorContextTester;
@@ -45,6 +47,7 @@
 import java.util.stream.IntStream;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatNoException;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -130,6 +133,56 @@ void testMergeHighlightingTokens() {
 
   }
 
+  @ParameterizedTest(name = "{0}")
+  @ValueSource(strings = {
+    // Tab-indented query: ensures tabs producing position differences do not crash highlighting.
+    "highlightLongQuery.bsl",
+    // СГРУППИРОВАТЬ ПО on separate BSL continuation lines -> single multiline SDBL token.
+    "highlightCrmQuery.bsl",
+    // ИНДЕКСИРОВАТЬ ПО on separate lines (reproduces PR #424 comment from ERP 2.5 module).
+    "highlightErpIndexByQuery.bsl",
+    // Real-world reproducer from issue #318 (CRM_КлиентыСервер.bsl attachment).
+    "CRM_КлиентыСервер.bsl"
+  })
+  void testHighlightingDoesNotThrowOnFile(String fileName) {
+    // given
+    context = SensorContextTester.create(Path.of("."));
+    highlighter = new BSLHighlighter(context);
+    var baseDirName = "src/test/resources/examples";
+    var path = Path.of(baseDirName, fileName);
+    documentContext = BSLLSBinding.getServerContext().addDocument(path.toUri());
+    BSLLSBinding.getServerContext().rebuildDocument(documentContext);
+    inputFile = Tools.inputFileBSL(fileName, Path.of(baseDirName).toFile());
+
+    // when/then - should not throw, even for multiline SDBL tokens or tab-indented queries
+    assertThatNoException().isThrownBy(() ->
+      highlighter.saveHighlighting(inputFile, documentContext)
+    );
+  }
+
+  @Test
+  void testSaveHighlightingWithInvalidTokenPosition() {
+    // given
+    context = SensorContextTester.create(Path.of("."));
+    highlighter = new BSLHighlighter(context);
+    documentContext = mock(DocumentContext.class);
+
+    // Create a token with position exceeding line length
+    var token = new CommonToken(BSLLexer.IF_KEYWORD, "Если");
+    token.setLine(1);
+    token.setCharPositionInLine(20);
+
+    when(documentContext.getTokens()).thenReturn(List.of(token));
+
+    // Create InputFile with short content (line has less than 20 characters)
+    inputFile = Tools.inputFileBSL(FILE_NAME, BASE_DIR, "А = 1;");
+
+    // when/then - should not throw
+    assertThatNoException().isThrownBy(() ->
+      highlighter.saveHighlighting(inputFile, documentContext)
+    );
+  }
+
   private void testHighlighting(Vocabulary vocabulary, Map<String, TypeOfText> highlightingMap) {
     // given
     initContext(vocabulary);