Merge branch 'chanicpanic-terminal-priorities-earley'

Author: erezsh

CHANGELOG.md

@@ -8,4 +8,6 @@ v1.0
 
 - `v_args(meta=True)` now gives meta as the first argument. i.e. `(meta, children)`
 
-- Renamed TraditionalLexer to BasicLexer, and 'standard' lexer option to 'basic'
+- Renamed TraditionalLexer to BasicLexer, and 'standard' lexer option to 'basic'
+
+- Default priority is now 0, for both terminals and rules (used to be 1 for terminals)

docs/grammar.md

@@ -99,11 +99,13 @@ num_list: "[" _separated{NUMBER, ","} "]"   // Will match "[1, 2, 3]" etc.
 
 ### Priority
 
-Terminals can be assigned priority only when using a lexer (future versions may support Earley's dynamic lexing).
+Terminals can be assigned a priority to influence lexing. Terminal priorities
+are signed integers with a default value of 0.
 
-Priority can be either positive or negative. If not specified for a terminal, it defaults to 1.
+When using a lexer, the highest priority terminals are always matched first.
 
-Highest priority terminals are always matched first.
+When using Earley's dynamic lexing, terminal priorities are used to prefer
+certain lexings and resolve ambiguity.
 
 ### Regexp Flags
 
@@ -228,9 +230,12 @@ four_words: word ~ 4
 
 ### Priority
 
-Rules can be assigned priority only when using Earley (future versions may support LALR as well).
+Like terminals, rules can be assigned a priority. Rule priorities are signed
+integers with a default value of 0.
 
-Priority can be either positive or negative. In not specified for a terminal, it's assumed to be 1 (i.e. the default).
+When using LALR, the highest priority rules are used to resolve collision errors.
+
+When using Earley, rule priorities are used to resolve ambiguity.
 
 <a name="dirs"></a>
 ## Directives
@@ -321,4 +326,4 @@ Can also be used to implement a plugin system where a core grammar is extended b
 %extend NUMBER: /0x\w+/
 ```
 
-For both `%extend` and `%override`, there is not requirement for a rule/terminal to come from another file, but that is probably the most common usecase
+For both `%extend` and `%override`, there is not requirement for a rule/terminal to come from another file, but that is probably the most common usecase

lark/lark.py

@@ -73,7 +73,7 @@ class LarkOptions(Serialize):
     start
             The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start")
     debug
-            Display debug information and extra warnings. Use only when debugging (default: False)
+            Display debug information and extra warnings. Use only when debugging (Default: ``False``)
             When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed.
     transformer
             Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster)
@@ -95,7 +95,7 @@ class LarkOptions(Serialize):
     g_regex_flags
             Flags that are applied to all terminals (both regex and strings)
     keep_all_tokens
-            Prevent the tree builder from automagically removing "punctuation" tokens (default: False)
+            Prevent the tree builder from automagically removing "punctuation" tokens (Default: ``False``)
     tree_class
             Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``.
 
@@ -123,13 +123,13 @@ class LarkOptions(Serialize):
     **=== Misc. / Domain Specific Options ===**
 
     postlex
-            Lexer post-processing (Default: None) Only works with the basic and contextual lexers.
+            Lexer post-processing (Default: ``None``) Only works with the basic and contextual lexers.
     priority
-            How priorities should be evaluated - auto, none, normal, invert (Default: auto)
+            How priorities should be evaluated - "auto", ``None``, "normal", "invert" (Default: "auto")
     lexer_callbacks
             Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
     use_bytes
-            Accept an input of type ``bytes`` instead of ``str`` (Python 3 only).
+            Accept an input of type ``bytes`` instead of ``str``.
     edit_terminals
             A callback for editing the terminals before parse.
     import_paths
@@ -391,13 +391,17 @@ def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None:
             for rule in self.rules:
                 if rule.options.priority is not None:
                     rule.options.priority = -rule.options.priority
+            for term in self.terminals:
+                term.priority = -term.priority
         # Else, if the user asked to disable priorities, strip them from the
-        # rules. This allows the Earley parsers to skip an extra forest walk
+        # rules and terminals. This allows the Earley parsers to skip an extra forest walk
         # for improved performance, if you don't need them (or didn't specify any).
         elif self.options.priority is None:
             for rule in self.rules:
                 if rule.options.priority is not None:
                     rule.options.priority = None
+            for term in self.terminals:
+                term.priority = 0
 
         # TODO Deprecate lexer_callbacks?
         self.lexer_conf = LexerConf(

lark/parser_frontends.py

@@ -162,8 +162,6 @@ class EarleyRegexpMatcher:
     def __init__(self, lexer_conf):
         self.regexps = {}
         for t in lexer_conf.terminals:
-            if t.priority:
-                raise GrammarError("Dynamic Earley doesn't support weights on terminals", t, t.priority)
             regexp = t.pattern.to_regexp()
             try:
                 width = get_regexp_width(regexp)[0]
@@ -186,13 +184,13 @@ def create_earley_parser__dynamic(lexer_conf, parser_conf, options=None, **kw):
         raise GrammarError("Earley's dynamic lexer doesn't support lexer_callbacks.")
 
     earley_matcher = EarleyRegexpMatcher(lexer_conf)
-    return xearley.Parser(parser_conf, earley_matcher.match, ignore=lexer_conf.ignore, **kw)
+    return xearley.Parser(lexer_conf, parser_conf, earley_matcher.match, **kw)
 
 def _match_earley_basic(term, token):
     return term.name == token.type
 
 def create_earley_parser__basic(lexer_conf, parser_conf, options, **kw):
-    return earley.Parser(parser_conf, _match_earley_basic, **kw)
+    return earley.Parser(lexer_conf, parser_conf, _match_earley_basic, **kw)
 
 def create_earley_parser(lexer_conf, parser_conf, options):
     resolve_ambiguity = options.ambiguity == 'resolve'

lark/parsers/earley.py

@@ -11,17 +11,19 @@
 
 from collections import deque
 
+from ..lexer import Token
 from ..tree import Tree
 from ..exceptions import UnexpectedEOF, UnexpectedToken
 from ..utils import logger
 from .grammar_analysis import GrammarAnalyzer
 from ..grammar import NonTerminal
 from .earley_common import Item, TransitiveItem
-from .earley_forest import ForestSumVisitor, SymbolNode, ForestToParseTree
+from .earley_forest import ForestSumVisitor, SymbolNode, TokenNode, ForestToParseTree
 
 class Parser:
-    def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, debug=False, tree_class=Tree):
+    def __init__(self, lexer_conf, parser_conf, term_matcher, resolve_ambiguity=True, debug=False, tree_class=Tree):
         analysis = GrammarAnalyzer(parser_conf)
+        self.lexer_conf = lexer_conf
         self.parser_conf = parser_conf
         self.resolve_ambiguity = resolve_ambiguity
         self.debug = debug
@@ -42,13 +44,21 @@ def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, debug=Fals
             if rule.origin not in self.predictions:
                 self.predictions[rule.origin] = [x.rule for x in analysis.expand_rule(rule.origin)]
 
-            ## Detect if any rules have priorities set. If the user specified priority = "none" then
-            #  the priorities will be stripped from all rules before they reach us, allowing us to
+            ## Detect if any rules/terminals have priorities set. If the user specified priority = None, then
+            #  the priorities will be stripped from all rules/terminals before they reach us, allowing us to
             #  skip the extra tree walk. We'll also skip this if the user just didn't specify priorities
-            #  on any rules.
+            #  on any rules/terminals.
             if self.forest_sum_visitor is None and rule.options.priority is not None:
                 self.forest_sum_visitor = ForestSumVisitor
 
+        # Check terminals for priorities
+        # Ignore terminal priorities if the basic lexer is used
+        if self.lexer_conf.lexer_type != 'basic' and self.forest_sum_visitor is None:
+            for term in self.lexer_conf.terminals:
+                if term.priority:
+                    self.forest_sum_visitor = ForestSumVisitor
+                    break
+
         self.term_matcher = term_matcher
 
 
@@ -232,8 +242,17 @@ def scan(i, token, to_scan):
                 if match(item.expect, token):
                     new_item = item.advance()
                     label = (new_item.s, new_item.start, i)
+                    # 'terminals' may not contain token.type when using %declare
+                    # Additionally, token is not always a Token
+                    # For example, it can be a Tree when using TreeMatcher
+                    term = terminals.get(token.type) if isinstance(token, Token) else None
+                    # Set the priority of the token node to 0 so that the
+                    # terminal priorities do not affect the Tree chosen by
+                    # ForestSumVisitor after the basic lexer has already
+                    # "used up" the terminal priorities
+                    token_node = TokenNode(token, term, priority=0)
                     new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
-                    new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token)
+                    new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token_node)
 
                     if new_item.expect in self.TERMINALS:
                         # add (B ::= Aai+1.B, h, y) to Q'
@@ -252,6 +271,8 @@ def scan(i, token, to_scan):
         # Define parser functions
         match = self.term_matcher
 
+        terminals = self.lexer_conf.terminals_by_name
+
         # Cache for nodes & tokens created in a particular parse step.
         transitives = [{}]
 

lark/parsers/earley_forest.py

@@ -36,13 +36,14 @@ class SymbolNode(ForestNode):
 
     Hence a Symbol Node with a single child is unambiguous.
 
-    :ivar s: A Symbol, or a tuple of (rule, ptr) for an intermediate node.
-    :ivar start: The index of the start of the substring matched by this
-        symbol (inclusive).
-    :ivar end: The index of the end of the substring matched by this
-        symbol (exclusive).
-    :ivar is_intermediate: True if this node is an intermediate node.
-    :ivar priority: The priority of the node's symbol.
+    Parameters:
+        s: A Symbol, or a tuple of (rule, ptr) for an intermediate node.
+        start: The index of the start of the substring matched by this symbol (inclusive).
+        end: The index of the end of the substring matched by this symbol (exclusive).
+
+    Properties:
+        is_intermediate: True if this node is an intermediate node.
+        priority: The priority of the node's symbol.
     """
     __slots__ = ('s', 'start', 'end', '_children', 'paths', 'paths_loaded', 'priority', 'is_intermediate', '_hash')
     def __init__(self, s, start, end):
@@ -113,11 +114,12 @@ class PackedNode(ForestNode):
     """
     A Packed Node represents a single derivation in a symbol node.
 
-    :ivar rule: The rule associated with this node.
-    :ivar parent: The parent of this node.
-    :ivar left: The left child of this node. ``None`` if one does not exist.
-    :ivar right: The right child of this node. ``None`` if one does not exist.
-    :ivar priority: The priority of this node.
+    Parameters:
+        rule: The rule associated with this node.
+        parent: The parent of this node.
+        left: The left child of this node. ``None`` if one does not exist.
+        right: The right child of this node. ``None`` if one does not exist.
+        priority: The priority of this node.
     """
     __slots__ = ('parent', 's', 'rule', 'start', 'left', 'right', 'priority', '_hash')
     def __init__(self, parent, s, rule, start, left, right):
@@ -172,6 +174,36 @@ def __repr__(self):
             symbol = self.s.name
         return "({}, {}, {}, {})".format(symbol, self.start, self.priority, self.rule.order)
 
+class TokenNode(ForestNode):
+    """
+    A Token Node represents a matched terminal and is always a leaf node.
+
+    Parameters:
+        token: The Token associated with this node.
+        term: The TerminalDef matched by the token.
+        priority: The priority of this node.
+    """
+    __slots__ = ('token', 'term', 'priority', '_hash')
+    def __init__(self, token, term, priority=None):
+        self.token = token
+        self.term = term
+        if priority is not None:
+            self.priority = priority
+        else:
+            self.priority = term.priority if term is not None else 0
+        self._hash = hash(token)
+
+    def __eq__(self, other):
+        if not isinstance(other, TokenNode):
+            return False
+        return self is other or (self.token == other.token)
+
+    def __hash__(self):
+        return self._hash
+
+    def __repr__(self):
+        return repr(self.token)
+
 class ForestVisitor:
     """
     An abstract base class for building forest visitors.
@@ -187,7 +219,8 @@ class ForestVisitor:
     methods. Returning a node(s) will schedule them to be visited. The visitor
     will begin to backtrack if no nodes are returned.
 
-    :ivar single_visit: If ``True``, non-Token nodes will only be visited once.
+    Parameters:
+        single_visit: If ``True``, non-Token nodes will only be visited once.
     """
 
     def __init__(self, single_visit=False):
@@ -224,11 +257,12 @@ def visit_packed_node_out(self, node):
     def on_cycle(self, node, path):
         """Called when a cycle is encountered.
 
-        :param node: The node that causes a cycle.
-        :param path: The list of nodes being visited: nodes that have been
-            entered but not exited. The first element is the root in a forest
-            visit, and the last element is the node visited most recently.
-            ``path`` should be treated as read-only.
+        Parameters:
+            node: The node that causes a cycle.
+            path: The list of nodes being visited: nodes that have been
+                entered but not exited. The first element is the root in a forest
+                visit, and the last element is the node visited most recently.
+                ``path`` should be treated as read-only.
         """
         pass
 
@@ -291,8 +325,8 @@ def visit(self, root):
                 input_stack.append(next_node)
                 continue
 
-            if not isinstance(current, ForestNode):
-                vtn(current)
+            if isinstance(current, TokenNode):
+                vtn(current.token)
                 input_stack.pop()
                 continue
 
@@ -322,8 +356,7 @@ def visit(self, root):
                 if next_node is None:
                     continue
 
-                if not isinstance(next_node, ForestNode) and \
-                        not isinstance(next_node, Token):
+                if not isinstance(next_node, ForestNode):
                     next_node = iter(next_node)
                 elif id(next_node) in visiting:
                     oc(next_node, path)
@@ -491,15 +524,13 @@ class ForestToParseTree(ForestTransformer):
     """Used by the earley parser when ambiguity equals 'resolve' or
     'explicit'. Transforms an SPPF into an (ambiguous) parse tree.
 
-    tree_class: The tree class to use for construction
-    callbacks: A dictionary of rules to functions that output a tree
-    prioritizer: A ``ForestVisitor`` that manipulates the priorities of
-        ForestNodes
-    resolve_ambiguity: If True, ambiguities will be resolved based on
-        priorities. Otherwise, `_ambig` nodes will be in the resulting
-        tree.
-    use_cache: If True, the results of packed node transformations will be
-        cached.
+    Parameters:
+        tree_class: The tree class to use for construction
+        callbacks: A dictionary of rules to functions that output a tree
+        prioritizer: A ``ForestVisitor`` that manipulates the priorities of ForestNodes
+        resolve_ambiguity: If True, ambiguities will be resolved based on
+                        priorities. Otherwise, `_ambig` nodes will be in the resulting tree.
+        use_cache: If True, the results of packed node transformations will be cached.
     """
 
     def __init__(self, tree_class=Tree, callbacks=dict(), prioritizer=ForestSumVisitor(), resolve_ambiguity=True, use_cache=True):
@@ -643,21 +674,18 @@ class TreeForestTransformer(ForestToParseTree):
     Non-tree transformations are made possible by override of
     ``__default__``, ``__default_token__``, and ``__default_ambig__``.
 
-    .. note::
-
+    Note:
         Tree shaping features such as inlined rules and token filtering are
-        not built into the transformation. Positions are also not
-        propagated.
-
-    :param tree_class: The tree class to use for construction
-    :param prioritizer: A ``ForestVisitor`` that manipulates the priorities of
-        nodes in the SPPF.
-    :param resolve_ambiguity: If True, ambiguities will be resolved based on
-        priorities.
-    :param use_cache: If True, caches the results of some transformations,
-        potentially improving performance when ``resolve_ambiguity==False``.
-        Only use if you know what you are doing: i.e. All transformation
-        functions are pure and referentially transparent.
+        not built into the transformation. Positions are also not propagated.
+
+    Parameters:
+        tree_class: The tree class to use for construction
+        prioritizer: A ``ForestVisitor`` that manipulates the priorities of nodes in the SPPF.
+        resolve_ambiguity: If True, ambiguities will be resolved based on priorities.
+        use_cache (bool): If True, caches the results of some transformations,
+                          potentially improving performance when ``resolve_ambiguity==False``.
+                          Only use if you know what you are doing: i.e. All transformation
+                          functions are pure and referentially transparent.
     """
 
     def __init__(self, tree_class=Tree, prioritizer=ForestSumVisitor(), resolve_ambiguity=True, use_cache=False):