Lesser changes and possible bugs encountered adding location information #1413 (#1414)

These are changes and slight bugs encounted in PR #1413

As a separate PR, this should reduce the size and effort for dealing with #1413, which I'll rebase after this is merged

Author: rocky

mathics/builtin/patterns/basic.py

@@ -10,6 +10,7 @@
 from mathics.core.builtin import PatternObject
 from mathics.core.evaluation import Evaluation
 from mathics.core.expression import Expression
+from mathics.core.symbols import BaseElement
 
 # This tells documentation how to sort this module
 sort_order = "mathics.builtin.rules-and-patterns.basic"
@@ -89,7 +90,7 @@ class Blank(_Blank):
     }
     summary_text = "match to any single expression"
 
-    def match(self, expression: Expression, pattern_context: dict):
+    def match(self, expression: BaseElement, pattern_context: dict):
         vars_dict = pattern_context["vars_dict"]
         yield_func = pattern_context["yield_func"]
 

mathics/core/definitions.py

@@ -18,7 +18,6 @@
 from mathics.core.attributes import A_NO_ATTRIBUTES
 from mathics.core.convert.expression import to_mathics_list
 from mathics.core.element import BaseElement, fully_qualified_symbol_name
-from mathics.core.load_builtin import definition_contribute, mathics3_builtins_modules
 from mathics.core.rules import BaseRule, Rule
 from mathics.core.symbols import Atom, Symbol, strip_context
 from mathics.core.util import canonic_filename
@@ -1066,6 +1065,10 @@ def load_builtin_definitions(
     """
     Load definitions from Builtin classes, autoload files and extension modules.
     """
+    from mathics.core.load_builtin import (
+        definition_contribute,
+        mathics3_builtins_modules,
+    )
     from mathics.eval.files_io.files import get_file_time
     from mathics.eval.pymathics import PyMathicsLoadException, load_pymathics_module
     from mathics.session import autoload_files

mathics/core/expression.py

@@ -1193,7 +1193,7 @@ def eval_range(indices):
                 recompute_properties = False
                 for index in indices:
                     element = elements[index]
-                    if not element.has_form("Unevaluated", 1):
+                    if not (element.is_literal or element.has_form("Unevaluated", 1)):
                         if isinstance(element, EvalMixin):
                             new_value = element.evaluate(evaluation)
                             # We need id() because != by itself is too permissive
@@ -1209,7 +1209,7 @@ def rest_range(indices):
                         return
                     for index in indices:
                         element = elements[index]
-                        if element.has_form("Evaluate", 1):
+                        if not element.is_literal and element.has_form("Evaluate", 1):
                             if isinstance(element, EvalMixin):
                                 new_value = element.evaluate(evaluation)
                                 # We need id() because != by itself is too permissive

mathics/core/parser/convert.py

@@ -10,6 +10,7 @@
 
 from mathics.core.atoms import Integer, MachineReal, PrecisionReal, Rational, String
 from mathics.core.convert.expression import to_expression, to_mathics_list
+from mathics.core.element import BaseElement
 from mathics.core.number import RECONSTRUCT_MACHINE_PRECISION_DIGITS
 from mathics.core.parser.ast import (
     Filename as AST_Filename,
@@ -189,7 +190,7 @@ class Converter(GenericConverter):
     def __init__(self):
         self.definitions = None
 
-    def convert(self, node, definitions):
+    def convert(self, node, definitions) -> BaseElement:
         self.definitions = definitions
         result = self.do_convert(node)
         self.definitions = None

mathics/core/parser/parser.py

@@ -49,7 +49,7 @@
 # FIXME: should rework so we don't have to do this
 # We have the character name ImaginaryI and ImaginaryJ, but we should
 # have the *operator* name, "I".
-special_symbols["\uF74F"] = special_symbols["\uF74E"] = "I"
+special_symbols["\uf74f"] = special_symbols["\uf74e"] = "I"
 
 
 # An operator precedence value that will ensure that whatever operator

mathics/core/parser/util.py

@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 
-from typing import Any, FrozenSet, Tuple
+from typing import FrozenSet, Optional, Tuple
 
 from mathics_scanner.feed import LineFeeder
 
+from mathics.core.definitions import Definitions
+from mathics.core.element import BaseElement
 from mathics.core.parser.convert import convert
 from mathics.core.parser.feed import MathicsSingleLineFeeder
 from mathics.core.parser.parser import Parser
@@ -12,7 +14,7 @@
 parser = Parser()
 
 
-def parse(definitions, feeder: LineFeeder) -> Any:
+def parse(definitions, feeder: LineFeeder) -> Optional[BaseElement]:
     """
     Parse input (from the frontend, -e, input files, ToExpression etc).
     Look up symbols according to the Definitions instance supplied.
@@ -22,7 +24,9 @@ def parse(definitions, feeder: LineFeeder) -> Any:
     return parse_returning_code(definitions, feeder)[0]
 
 
-def parse_incrementally_by_line(definitions, feeder: LineFeeder) -> Any:
+def parse_incrementally_by_line(
+    definitions: Definitions, feeder: LineFeeder
+) -> Optional[BaseElement]:
     """Parse input incrementally by line. This is in contrast to parse() or
     parser_returning_code(), which parse the *entire*
     input which could be many line.
@@ -50,19 +54,28 @@ def parse_incrementally_by_line(definitions, feeder: LineFeeder) -> Any:
     return convert(ast, definitions)
 
 
-def parse_returning_code(definitions, feeder: LineFeeder) -> Tuple[Any, str]:
-    """
-    Parse input (from the frontend, -e, input files, ToExpression etc).
+def parse_returning_code(
+    definitions: Definitions, feeder: LineFeeder
+) -> Tuple[Optional[BaseElement], str]:
+    """Parse input (from the frontend, -e, input files, ToExpression etc).
     Look up symbols according to the Definitions instance supplied.
 
-    Feeder must implement the feed and empty methods, see core/parser/feed.py.
+    ``feeder`` must implement the ``feed()`` and ``empty()``
+    methods. See the mathics_scanner.feed module.
+
     """
+    from mathics.core.expression import Expression
+
     ast = parser.parse(feeder)
-    source_code = parser.tokeniser.code if hasattr(parser.tokeniser, "code") else ""
-    if ast is not None:
-        return convert(ast, definitions), source_code
-    else:
-        return None, source_code
+
+    source_text = parser.tokeniser.source_text
+
+    if ast is None:
+        return None, source_text
+
+    converted = convert(ast, definitions)
+
+    return converted, source_text
 
 
 class SystemDefinitions:

mathics/core/pattern.py

@@ -399,6 +399,12 @@ def get_match_count(self, vars_dict: Optional[dict] = None) -> Tuple[int, int]:
         """The number of matches"""
         return (1, 1)
 
+    @property
+    def short_name(self) -> str:
+        return (
+            self.atom.short_name if hasattr(self.atom, "short_name") else str(self.atom)
+        )
+
 
 # class StopGenerator_ExpressionPattern_match(StopGenerator):
 #    pass

mathics/core/symbols.py

@@ -495,6 +495,8 @@ def evaluate(self, evaluation):
         for rule in rules:
             result = rule.apply(self, evaluation, fully=True)
             if result is not None and not result.sameQ(self):
+                if result.is_literal:
+                    return result
                 return result.evaluate(evaluation)
         return self
 
@@ -743,7 +745,12 @@ class BooleanType(SymbolConstant):
     the constant is either SymbolTrue or SymbolFalse.
     """
 
-    pass
+    @property
+    def is_literal(self) -> bool:
+        """
+        We don't allow changing Boolean values True and False.
+        """
+        return True
 
 
 def symbol_set(*symbols: Symbol) -> FrozenSet[Symbol]:

mathics/eval/tracing.py

@@ -26,8 +26,18 @@ def skip_trivial_evaluation(expr, status: str, orig_expr=None) -> bool:
     * the evaluation is a literal that evaluates to the same thing,
     * evaluating a Symbol which the Symbol.
     * Showing the return value of a ListExpression literal
+    * Evaluating Pattern[] which define a pattern.
     """
+    from mathics.core.expression import Expression
     from mathics.core.symbols import Symbol, SymbolConstant
+    from mathics.core.systemsymbols import SymbolBlank, SymbolPattern
+
+    if isinstance(expr, tuple):
+        expr = expr[0]
+
+    if isinstance(expr, Expression):
+        if expr.head in (SymbolPattern, SymbolBlank):
+            return True
 
     if status == "Returning":
         if (
@@ -39,7 +49,9 @@ def skip_trivial_evaluation(expr, status: str, orig_expr=None) -> bool:
             return True
         pass
         if isinstance(expr, Symbol) and not isinstance(expr, SymbolConstant):
-            # Evaluation of a symbol, like Plus isn't that interesting
+            # Evaluation of a symbol, like Plus isn't that interesting.
+            # Right now, SymbolConstant are not literals. If this
+            # changes, we don't need this clause.
             return True
 
     else: