Merge pull request #1577 from chanicpanic/issue1576

Bugfix Earley: handle ambiguity from ignored tokens

Author: erezsh

lark/parsers/xearley.py

@@ -22,6 +22,7 @@
 from ..lexer import Token
 from ..grammar import Terminal
 from .earley import Parser as BaseParser
+from .earley_common import Item
 from .earley_forest import TokenNode
 
 if TYPE_CHECKING:
@@ -109,7 +110,14 @@ def scan(i, to_scan):
                     new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token_node)
                 else:
-                    new_item = item
+                    # Handle items carried over due to ignores
+                    new_item = Item(item.rule, item.ptr, item.start)
+                    label = (new_item.s, new_item.start, i + 1)
+                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
+                    if item.node:
+                        # new_item.node and item.node both represent the same symbol, so merge their children
+                        for child in item.node.children:
+                            new_item.node.add_family(new_item.s, child.rule, new_item.start, child.left, child.right)
 
                 if new_item.expect in self.TERMINALS:
                     # add (B ::= Aai+1.B, h, y) to Q'

tests/test_parser.py

@@ -784,6 +784,24 @@ def test_ambiguous_intermediate_node_conditionally_inlined_rule(self):
             self.assertEqual(ambig_tree.data, '_ambig')
             self.assertEqual(set(ambig_tree.children), expected)
 
+        @unittest.skipIf(LEXER=='basic', 'This ambiguity only occurs with the dynamic lexers')
+        def test_ambiguous_ignores(self):
+            grammar = """
+            !start: a "b"
+            !a: "a" | "a1" | "a12"
+            %ignore "1"
+            %ignore "2"
+            """
+
+            l = Lark(grammar, ambiguity='explicit', lexer=LEXER)
+            tree = l.parse('a12b')
+
+            expected = Tree('_ambig', [
+                Tree('start', [Tree('a', ['a']), 'b']),
+                Tree('start', [Tree('a', ['a1']), 'b']),
+                Tree('start', [Tree('a', ['a12']), 'b']),
+            ])
+            self.assertEqual(tree, expected)
 
         @unittest.skipIf(LEXER=='basic', "Requires dynamic lexer")
         def test_fruitflies_ambig(self):
@@ -876,6 +894,26 @@ def test_multiple_start_solutions(self):
             tree = l.parse('x')
             assert tree == Tree('start', [Tree('a', ['x'])])
 
+        @unittest.skipIf(LEXER=='basic', 'This scenario only occurs with the dynamic lexers')
+        def test_multiple_start_solutions2(self):
+            grammar = r"""
+                !start: "foo1" | "foo" | "foo12"
+                %ignore "1"
+                %ignore "2"
+            """
+            l = Lark(grammar, ambiguity='explicit', lexer=LEXER)
+            tree = l.parse('foo12')
+
+            expected = Tree('_ambig', [
+                Tree('start', ['foo1']),
+                Tree('start', ['foo']),
+                Tree('start', ['foo12']),
+            ])
+            self.assertEqual(tree, expected)
+
+            l = Lark(grammar, ambiguity='resolve', lexer=LEXER)
+            tree = l.parse('foo12')
+            self.assertEqual(tree, Tree('start', ['foo1']))
 
         def test_cycle(self):
             grammar = """