Merge pull request #113 from klauer/fix-end-position-of-comments-and-pragmas

Fix end position of comments and pragmas

Author: klauer

.github/workflows/docs.yml

@@ -137,7 +137,7 @@ jobs:
           pip install docs-versions-menu
 
       - name: Download documentation artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: Documentation
 

blark/tests/test_parsing.py

@@ -2,9 +2,8 @@
 
 import pytest
 
-from blark.util import SourceType
-
 from ..parse import parse, parse_source_code, summarize
+from ..util import SourceType
 from . import conftest
 
 TEST_PATH = pathlib.Path(__file__).parent
@@ -97,3 +96,37 @@ def test_parsing_source(source_filename: str):
 def test_rule_smoke(grammar, name, value):
     result = conftest.get_grammar(start=name).parse(value)
     print(f"rule {name} value {value!r} into {result}")
+
+
+@pytest.mark.parametrize(
+    ("code", "expected_snippets"),
+    [
+        pytest.param(
+            """\
+(*
+123456789012345678
+*)
+// comment
+{pragma}
+VAR_GLOBAL
+    dummy : BOOL;
+END_VAR
+""",
+            [
+                "(*\n123456789012345678\n*)",
+                "// comment",
+                "{pragma}",
+            ],
+            id="issue_109",
+        ),
+    ],
+)
+def test_comment_parsing(code: str, expected_snippets: list[str]):
+    result = parse_source_code(code)
+
+    print(result.comments)
+
+    snippets = [
+        result.source_code[token.start_pos: token.end_pos] for token in result.comments
+    ]
+    assert snippets == expected_snippets

blark/util.py

@@ -363,7 +363,7 @@ def get_token(
         end_line: int,
         end_col: int,
     ) -> lark.Token:
-        block = text[start_pos:end_pos + 1]
+        block = text[start_pos:end_pos]
 
         if block.startswith("//"):
             type_ = "SINGLE_LINE_COMMENT"
@@ -424,7 +424,7 @@ def get_token(
                             start_pos,
                             start_line,
                             start_col,
-                            pos,
+                            pos + 1,
                             lineno,
                             colno + 1,
                         )
@@ -452,9 +452,9 @@ def get_token(
                             start_pos,
                             start_line,
                             start_col,
-                            pos + 1,  # two character ending
+                            pos + 2,  # two character ending
                             lineno,
-                            colno + 1,  # two character ending
+                            colno + 2,  # two character ending
                         )
                     )
                 skip = 1
@@ -466,7 +466,7 @@ def get_token(
                         pos,
                         lineno,
                         colno,
-                        pos + (len(lines[lineno]) - colno - 1),
+                        pos + (len(lines[lineno]) - colno),
                         lineno,
                         len(lines[lineno]),
                     )