Fill out location tracking

Author: rocky

mathics/core/expression.py

@@ -19,7 +19,7 @@
 )
 
 import sympy
-from mathics_scanner.location import SourceRange
+from mathics_scanner.location import SourceRange, SourceRange2
 
 from mathics.core.atoms import Integer1, String
 from mathics.core.attributes import (
@@ -277,7 +277,7 @@ class Expression(BaseElement, NumericOperators, EvalMixin):
     elements_properties: Optional[ElementsProperties]
     options: Optional[Dict[str, Any]]
     pattern_sequence: bool
-    location: Optional[Union[SourceRange, MethodType]]
+    location: Optional[Union[SourceRange, SourceRange2, MethodType]]
 
     def __init__(
         self,

mathics/core/parser/ast.py

@@ -23,14 +23,15 @@ class Node:
     expression's leaves and a non-leaf nodes.
     """
 
-    def __init__(self, head, *children):
+    def __init__(self, head, *children, location=None):
         if isinstance(head, Node):
             self.head = head
         else:
             self.head = Symbol(head)
         self.value = None
         self.children = list(children)
         self.parenthesised = False
+        self.location = location
 
     def get_head_name(self):
         if isinstance(self.head, Symbol):
@@ -68,11 +69,12 @@ class Atom(Node):
     their own. You can however compare Atoms for equality.
     """
 
-    def __init__(self, value):
+    def __init__(self, value, location=None):
         self.head = Symbol(self.__class__.__name__)
         self.value = value
         self.children = []
         self.parenthesised = False
+        self.location = location
 
     def __repr__(self):
         return "%s[%s]" % (self.head, self.value)
@@ -90,7 +92,13 @@ class Number(Atom):
     """
 
     def __init__(
-        self, value: str, sign: int = 1, base: int = 10, suffix=None, exp: int = 0
+        self,
+        value: str,
+        sign: int = 1,
+        base: int = 10,
+        suffix=None,
+        exp: int = 0,
+        location=None,
     ):
         assert isinstance(value, str)
         assert sign in (-1, 1)
@@ -104,6 +112,7 @@ def __init__(
         self.base = base
         self.suffix = suffix
         self.exp = exp
+        self.location = location
 
     def __repr__(self):
         result = self.value
@@ -132,10 +141,11 @@ class Symbol(Atom):
     are unique as they are say in Lisp, or Python.
     """
 
-    def __init__(self, value: str, context: Optional[str] = "System"):
+    def __init__(self, value: str, context: Optional[str] = "System", location=None):
         self.context = context
         self.value = value
         self.children = []
+        self.location = location
 
     # avoids recursive definition
     @property

mathics/core/parser/convert.py

@@ -194,7 +194,6 @@ def convert(self, node, definitions) -> BaseElement:
         self.definitions = definitions
         result = self.do_convert(node)
         self.definitions = None
-        self.location = None
         return result
 
     def do_convert(self, node):

mathics/core/parser/feed.py

@@ -21,12 +21,12 @@ def send_messages(self, evaluation) -> list:
 
 
 class MathicsSingleLineFeeder(SingleLineFeeder, MathicsLineFeeder):
-    "A feeder that feeds lines from an open ``File`` object"
+    "A feeder that feeds lines from an open location container object"
 
 
 class MathicsFileLineFeeder(FileLineFeeder, MathicsLineFeeder):
     "A feeder that feeds lines from an open ``File`` object"
 
 
 class MathicsMultiLineFeeder(MultiLineFeeder, MathicsLineFeeder):
-    "A feeder that feeds lines from an open ``File`` object"
+    "A feeder that feeds lines from an open contianer object"

mathics/core/parser/location.py

@@ -0,0 +1,116 @@
+"""
+Provides location tracking in parser.
+"""
+
+from typing import Callable, Optional
+
+import mathics_scanner
+from mathics_scanner.location import (
+    EVAL_METHODS,
+    ContainerKind,
+    SourceRange,
+    SourceRange2,
+)
+
+from mathics.core.parser.ast import Node
+
+
+def track_location(func: Callable) -> Callable:
+    """Python decorator for a parse method that adds location tracking
+    on non-leaf-nodes when TRACK_LOCATION is set. Otherwise, we just
+    run the parse method.
+
+    """
+
+    def wrapper(self, *args) -> Optional[Node]:
+        if not mathics_scanner.location.TRACK_LOCATIONS:
+            return func(self, *args)
+
+        # Save location information
+        # For expressions which make their way to
+        # FunctionApplyRule, saving a position here is
+        # extraneous because the FunctionApplyRule is
+        # the position.  But deal with this redundancy
+        # after the dust settles, and we have experience
+        # on what is desired.
+        start_column = self.tokeniser.pos
+        start_line = self.feeder.lineno
+        parsed_node = func(self, *args)
+
+        if parsed_node is not None and hasattr(parsed_node, "location"):
+            if self.feeder.container_kind == ContainerKind.PYTHON:
+                parsed_node.location = self.feeder.container
+                if self.feeder.container not in EVAL_METHODS:
+                    EVAL_METHODS.add(parsed_node.location)
+            else:
+                end_pos = self.tokeniser.pos
+                end_line = self.feeder.lineno
+                if start_line == end_line:
+                    parsed_node.location = SourceRange2(
+                        start_line=start_line,
+                        start_pos=start_column,
+                        end_pos=end_pos,
+                        container=self.feeder.container_index,
+                    )
+                else:
+                    parsed_node.location = SourceRange(
+                        start_line=start_line,
+                        start_pos=start_column,
+                        end_line=end_line,
+                        end_pos=end_pos,
+                        container=self.feeder.container_index,
+                    )
+
+        return parsed_node
+
+    return wrapper
+
+
+def track_token_location(func: Callable) -> Callable:
+    """Python decorator for a parse method that adds location
+    tracking to leaf-nodes, i.e. tokens. This happens though only TRACK_LOCATION is set. Otherwise, we just run the
+    normal parse method.
+
+    """
+
+    def wrapper(self, token) -> Optional[Node]:
+        if not mathics_scanner.location.TRACK_LOCATIONS:
+            return func(self, token)
+
+        # Save location information
+        # For expressions which make their way to
+        # FunctionApplyRule, saving a position here is
+        # extraneous because the FunctionApplyRule is
+        # the position.  But deal with this redundancy
+        # after the dust settles, and we have experience
+        # on what is desired.
+        start_column = token.pos
+        start_line = self.feeder.lineno
+        parsed_node = func(self, token)
+
+        if (
+            self.feeder.container_kind == ContainerKind.PYTHON
+            and self.feeder.container not in EVAL_METHODS
+        ):
+            location = self.feeder.container
+            EVAL_METHODS.add(location)
+
+        end_line = self.feeder.lineno
+        if start_line == end_line:
+            parsed_node.location = SourceRange2(
+                start_line=start_line,
+                start_pos=start_column,
+                end_pos=start_column + len(token.text) - 1,
+                container=self.feeder.container_index,
+            )
+        else:
+            parsed_node.location = SourceRange(
+                start_line=start_line,
+                start_pos=start_column,
+                end_line=self.feeder.lineno,
+                end_pos=start_column + len(token.text) - 1,
+                container=self.feeder.container_index,
+            )
+        return parsed_node
+
+    return wrapper