Revise API for position tracking

Author: rocky

examples/symbolic_logic/gries_schneider/test_gs.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+from mathics_scanner.location import ContainerKind
 
 from mathics.core.definitions import Definitions
 from mathics.core.evaluation import Evaluation
@@ -13,5 +14,8 @@
 for i in range(0, 4):
     evaluation = Evaluation(definitions=definitions, catch_interrupt=False)
 
-    expr = parse(definitions, MathicsSingleLineFeeder(f"<< GS{i}.m"))
+    expr = parse(
+        definitions,
+        MathicsSingleLineFeeder(f"<< GS{i}.m", "<test_gs>", ContainerKind.STRING),
+    )
     expr.evaluate(evaluation)

mathics/builtin/testing_expressions/string_tests.py

@@ -5,6 +5,7 @@
 import re
 
 from mathics_scanner import SingleLineFeeder, SyntaxError
+from mathics_scanner.location import ContainerKind
 
 from mathics.builtin.atomic.strings import anchor_pattern
 from mathics.core.atoms import Integer1, String
@@ -278,7 +279,7 @@ def eval(self, string, evaluation: Evaluation):
             )
             return
 
-        feeder = SingleLineFeeder(string.value)
+        feeder = SingleLineFeeder(string.value, "<SyntaxQ>", ContainerKind.STRING)
         try:
             parser.parse(feeder)
         except SyntaxError:

mathics/builtin/trace.py

@@ -22,16 +22,24 @@
 from time import time
 from typing import Callable
 
+import mathics_scanner.location
+
 import mathics.eval.tracing
 from mathics.core.attributes import A_HOLD_ALL, A_HOLD_ALL_COMPLETE, A_PROTECTED
-from mathics.core.builtin import Builtin
+from mathics.core.builtin import Builtin, Predefined
 from mathics.core.convert.python import from_bool, from_python
 from mathics.core.definitions import Definitions
 from mathics.core.evaluation import Evaluation
 from mathics.core.expression import Expression
 from mathics.core.list import ListExpression
 from mathics.core.rules import FunctionApplyRule
-from mathics.core.symbols import SymbolFalse, SymbolNull, SymbolTrue, strip_context
+from mathics.core.symbols import (
+    Symbol,
+    SymbolFalse,
+    SymbolNull,
+    SymbolTrue,
+    strip_context,
+)
 
 
 def traced_apply_function(
@@ -493,3 +501,33 @@ def eval(self, expr: Expression, evaluation: Evaluation):
         else:
             result = expr.evaluate(evaluation)
         return ListExpression(result, profile_result)
+
+
+class TrackLocations(Predefined):
+    r"""## <url>:TrackLocations native symbol:</url>
+
+    <dl>
+      <dt>'$TrackLocations'
+      <dd>specifies whether we should track \
+      source-text location information during evaluation. This \
+      can be helpful in debugging when there is a failure.
+    </dl>
+    """
+
+    name = "$TrackLocations"
+    messages = {"bool": "`1` should be True or False."}
+
+    summary_text = "track source-text locations in evaluation"
+
+    def evaluate(self, evaluation: Evaluation) -> Symbol:
+        print(mathics_scanner.position.MATHICS3_PATHS)
+        return from_bool(mathics_scanner.position.TRACK_LOCATIONS)
+
+    def eval_set(self, value, evaluation):
+        """Set[$TrackLocations, value_]"""
+        if value is SymbolTrue or value is SymbolFalse:
+            evaluation.definitions.set_ownvalue("System`$TrackLocations", value)
+            mathics.core.parser.parser.TRACK_LOCATIONS = value.to_python()
+        else:
+            evaluation.message("$TrackLocations", "bool", value)
+        return value

mathics/core/builtin.py

@@ -484,7 +484,15 @@ def get_functions(self, prefix="eval", is_pymodule=False):
                 definition_class = (
                     PyMathicsDefinitions() if is_pymodule else SystemDefinitions()
                 )
-                pattern = parse_builtin_rule(pattern, definition_class)
+
+                # Passing the function parameter is in a way
+                # redundant, because creating FunctionApplyRule has
+                # access to the function and sets the postion this
+                # way. But revised afte the dust has settled and
+                # we have a very good idea of what is desirable and useful.
+                pattern = parse_builtin_rule(
+                    pattern, definition_class, location=function
+                )
                 if unavailable_function:
                     function = unavailable_function
                 if attrs:

mathics/core/definitions.py

@@ -18,7 +18,6 @@
 from mathics.core.attributes import A_NO_ATTRIBUTES
 from mathics.core.convert.expression import to_mathics_list
 from mathics.core.element import BaseElement, fully_qualified_symbol_name
-from mathics.core.load_builtin import definition_contribute, mathics3_builtins_modules
 from mathics.core.rules import BaseRule, Rule
 from mathics.core.symbols import Atom, Symbol, strip_context
 from mathics.core.util import canonic_filename
@@ -1066,6 +1065,10 @@ def load_builtin_definitions(
     """
     Load definitions from Builtin classes, autoload files and extension modules.
     """
+    from mathics.core.load_builtin import (
+        definition_contribute,
+        mathics3_builtins_modules,
+    )
     from mathics.eval.files_io.files import get_file_time
     from mathics.eval.pymathics import PyMathicsLoadException, load_pymathics_module
     from mathics.session import autoload_files

mathics/core/expression.py

@@ -4,6 +4,7 @@
 import math
 from bisect import bisect_left
 from itertools import chain
+from types import MethodType
 from typing import (
     Any,
     Callable,
@@ -18,6 +19,7 @@
 )
 
 import sympy
+from mathics_scanner.location import SourceRange
 
 from mathics.core.atoms import Integer1, String
 from mathics.core.attributes import (
@@ -275,6 +277,7 @@ class Expression(BaseElement, NumericOperators, EvalMixin):
     elements_properties: Optional[ElementsProperties]
     options: Optional[Dict[str, Any]]
     pattern_sequence: bool
+    location: Optional[Union[SourceRange, MethodType]]
 
     def __init__(
         self,
@@ -300,6 +303,7 @@ def __init__(
 
         self._sequences = None
         self._cache = None
+        self.location = None
 
         # self.copy creates this
         self.original: Optional[Expression] = None

mathics/core/parser/convert.py

@@ -10,6 +10,7 @@
 
 from mathics.core.atoms import Integer, MachineReal, PrecisionReal, Rational, String
 from mathics.core.convert.expression import to_expression, to_mathics_list
+from mathics.core.element import BaseElement
 from mathics.core.number import RECONSTRUCT_MACHINE_PRECISION_DIGITS
 from mathics.core.parser.ast import (
     Filename as AST_Filename,
@@ -189,7 +190,7 @@ class Converter(GenericConverter):
     def __init__(self):
         self.definitions = None
 
-    def convert(self, node, definitions):
+    def convert(self, node, definitions) -> BaseElement:
         self.definitions = definitions
         result = self.do_convert(node)
         self.definitions = None

mathics/core/parser/parser.py

@@ -49,7 +49,7 @@
 # FIXME: should rework so we don't have to do this
 # We have the character name ImaginaryI and ImaginaryJ, but we should
 # have the *operator* name, "I".
-special_symbols["\uF74F"] = special_symbols["\uF74E"] = "I"
+special_symbols["\uf74f"] = special_symbols["\uf74e"] = "I"
 
 
 # An operator precedence value that will ensure that whatever operator

mathics/core/parser/util.py

@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
 
-from typing import Any, FrozenSet, Tuple
+from typing import FrozenSet, Optional, Tuple
 
 from mathics_scanner.feed import LineFeeder
+from mathics_scanner.location import ContainerKind, SourceRange
 
+from mathics.core.definitions import Definitions
+from mathics.core.element import BaseElement
 from mathics.core.parser.convert import convert
 from mathics.core.parser.feed import MathicsSingleLineFeeder
 from mathics.core.parser.parser import Parser
@@ -12,7 +15,7 @@
 parser = Parser()
 
 
-def parse(definitions, feeder: LineFeeder) -> Any:
+def parse(definitions, feeder: LineFeeder) -> BaseElement:
     """
     Parse input (from the frontend, -e, input files, ToExpression etc).
     Look up symbols according to the Definitions instance supplied.
@@ -22,7 +25,9 @@ def parse(definitions, feeder: LineFeeder) -> Any:
     return parse_returning_code(definitions, feeder)[0]
 
 
-def parse_incrementally_by_line(definitions, feeder: LineFeeder) -> Any:
+def parse_incrementally_by_line(
+    definitions: Definitions, feeder: LineFeeder
+) -> Optional[BaseElement]:
     """Parse input incrementally by line. This is in contrast to parse() or
     parser_returning_code(), which parse the *entire*
     input which could be many line.
@@ -50,19 +55,40 @@ def parse_incrementally_by_line(definitions, feeder: LineFeeder) -> Any:
     return convert(ast, definitions)
 
 
-def parse_returning_code(definitions, feeder: LineFeeder) -> Tuple[Any, str]:
-    """
-    Parse input (from the frontend, -e, input files, ToExpression etc).
+def parse_returning_code(
+    definitions: Definitions, feeder: LineFeeder
+) -> Tuple[BaseElement, str]:
+    """Parse input (from the frontend, -e, input files, ToExpression etc).
     Look up symbols according to the Definitions instance supplied.
 
-    Feeder must implement the feed and empty methods, see core/parser/feed.py.
+    ``feeder`` must implement the ``feed()`` and ``empty()``
+    methods. See the mathics_scanner.feed module.
+
     """
+    from mathics.core.expression import Expression
+
     ast = parser.parse(feeder)
-    source_code = parser.tokeniser.code if hasattr(parser.tokeniser, "code") else ""
-    if ast is not None:
-        return convert(ast, definitions), source_code
-    else:
-        return None, source_code
+
+    # For expressions which make their way to
+    # FunctionApplyRule, saving a position here is
+    # extraneous because teh FunctionApplyRule an get
+    # the position.  But deal with this redundancy
+    # after the dust settles, and we have experience
+    # on what is desired.
+    location = (
+        feeder.container
+        if feeder.container_kind == ContainerKind.PYTHON
+        else SourceRange(0, parser.tokeniser.pos, feeder.container_index)
+    )
+    source_text = parser.tokeniser.source_text
+
+    if ast is None:
+        return None, source_text
+
+    converted = convert(ast, definitions)
+    if isinstance(converted, Expression):
+        converted.location = location
+    return converted, source_text
 
 
 class SystemDefinitions:
@@ -111,9 +137,12 @@ def lookup_name(self, name):
         return ensure_context(name, context)
 
 
-def parse_builtin_rule(string, definitions=SystemDefinitions()):
+def parse_builtin_rule(string, definitions=SystemDefinitions(), location=None):
     """
     Parse rules specified in builtin docstrings/attributes. Every symbol
     in the input is created in the System` context.
     """
-    return parse(definitions, MathicsSingleLineFeeder(string, "<builtin_rules>"))
+    return parse(
+        definitions,
+        MathicsSingleLineFeeder(string, location, ContainerKind.PYTHON),
+    )

mathics/core/pattern.py

@@ -399,6 +399,10 @@ def get_match_count(self, vars_dict: Optional[dict] = None) -> Tuple[int, int]:
         """The number of matches"""
         return (1, 1)
 
+    @property
+    def short_name(self) -> str:
+        return self.atom.short_name
+
 
 # class StopGenerator_ExpressionPattern_match(StopGenerator):
 #    pass
@@ -421,6 +425,7 @@ def __init__(
         evaluation: Optional[Evaluation] = None,
     ):
         self.expr = expr
+        self.location = expr.location if hasattr(expr, "location") else None
         head = expr.head
         if attributes is None and evaluation:
             attributes = head.get_attributes(evaluation.definitions)