Revise API for position tracking (#1413)

Start tracking locations in specific expressions to aid error reporting
and debugging.

We need to take care to avoid bloating memory with positions.

Right now, start locations are tagged for FunctionApplyRule, an ExpressionPattern -> Python method. Here, the location is the Python method, or rather, can be extracted from that. So we can reuse the location information that Python is storing anyway.

For Expressions and ExpressionPatterns that are read in from Files, Things are more complicated..

This needs to be coordinated with a change in mathics-scanner with the same git branch name.

Author: rocky

.github/workflows/consistency-checks.yml

@@ -23,7 +23,7 @@ jobs:
         sudo apt update -qq && sudo apt install llvm-dev remake
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
 
     - name: Install Mathics with minimum dependencies

.github/workflows/macos.yml

@@ -35,7 +35,7 @@ jobs:
         cd ..
         # We can comment out after next Mathics-Scanner release
         # python -m pip install Mathics-Scanner[full]
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
         remake -x develop-full
     - name: Test Mathics3

.github/workflows/mypy.yml

@@ -27,7 +27,7 @@ jobs:
       run: |
         pip install mypy==1.13 sympy==1.12
         # Adjust below for right branch
-        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+        git clone --depth 1 --branch position-tracking https://github.com/Mathics3/mathics-scanner
         cd mathics-scanner/
         pip install -e .
         bash ./admin-tools/make-JSON-tables.sh

.github/workflows/packages.yml

@@ -25,7 +25,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
     - name: Run Mathics3 Combinatorica tests
       run: |
         git submodule init

.github/workflows/pyodide.yml

@@ -55,7 +55,7 @@ jobs:
           pip install "setuptools>=70.0.0" PyYAML click packaging pytest
 
           # We can comment out after next Mathics-Scanner release
-          python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner
+          python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner
           # pip install --no-build-isolation -e .
           # cd ..
 

.github/workflows/ubuntu-cython.yml

@@ -30,7 +30,7 @@ jobs:
         pip install -e .
         cd ..
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
         cd ..
 

.github/workflows/ubuntu.yml

@@ -31,7 +31,7 @@ jobs:
         cd ..
         # We can comment out after next Mathics-Scanner release
         # python -m pip install Mathics-Scanner[full]
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
         remake -x develop-full
     - name: Test Mathics

.github/workflows/windows.yml

@@ -39,7 +39,7 @@ jobs:
         pip install -e .
         cd ..
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
 
         # python -m pip install Mathics-Scanner[full]

examples/symbolic_logic/gries_schneider/test_gs.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+from mathics_scanner.location import ContainerKind
 
 from mathics.core.definitions import Definitions
 from mathics.core.evaluation import Evaluation
@@ -13,5 +14,8 @@
 for i in range(0, 4):
     evaluation = Evaluation(definitions=definitions, catch_interrupt=False)
 
-    expr = parse(definitions, MathicsSingleLineFeeder(f"<< GS{i}.m"))
+    expr = parse(
+        definitions,
+        MathicsSingleLineFeeder(f"<< GS{i}.m", "<test_gs>", ContainerKind.STRING),
+    )
     expr.evaluate(evaluation)

mathics/builtin/testing_expressions/string_tests.py

@@ -5,6 +5,7 @@
 import re
 
 from mathics_scanner import SingleLineFeeder, SyntaxError
+from mathics_scanner.location import ContainerKind
 
 from mathics.builtin.atomic.strings import anchor_pattern
 from mathics.core.atoms import Integer1, String
@@ -278,7 +279,7 @@ def eval(self, string, evaluation: Evaluation):
             )
             return
 
-        feeder = SingleLineFeeder(string.value)
+        feeder = SingleLineFeeder(string.value, "<SyntaxQ>", ContainerKind.STRING)
         try:
             parser.parse(feeder)
         except SyntaxError: