Bugfix Earley: handle ambiguity from strings ending in ignored tokens

Author: chanicpanic

lark/parsers/xearley.py

@@ -22,6 +22,7 @@
 from ..lexer import Token
 from ..grammar import Terminal
 from .earley import Parser as BaseParser
+from .earley_common import Item
 from .earley_forest import TokenNode
 
 if TYPE_CHECKING:
@@ -108,6 +109,14 @@ def scan(i, to_scan):
                     token_node = TokenNode(token, terminals[token.type])
                     new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token_node)
+                elif item.is_complete and item.s == start_symbol:
+                    # Handle completed start symbols
+                    new_item = Item(item.rule, item.ptr, item.start)
+                    label = (new_item.s, new_item.start, i + 1)
+                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
+                    # new_item.node and item.node both represent the start symbol, so merge their children
+                    for child in item.node.children:
+                        new_item.node.add_family(new_item.s, child.rule, new_item.start, child.left, child.right)
                 else:
                     new_item = item
 

tests/test_parser.py

@@ -876,6 +876,26 @@ def test_multiple_start_solutions(self):
             tree = l.parse('x')
             assert tree == Tree('start', [Tree('a', ['x'])])
 
+        @unittest.skipIf(LEXER=='basic', 'This scenario only occurs with the dynamic lexers')
+        def test_multiple_start_solutions2(self):
+            grammar = r"""
+                !start: "foo1" | "foo" | "foo12"
+                %ignore "1"
+                %ignore "2"
+            """
+            l = Lark(grammar, ambiguity='explicit', lexer=LEXER)
+            tree = l.parse('foo12')
+
+            expected = Tree('_ambig', [
+                Tree('start', ['foo1']),
+                Tree('start', ['foo']),
+                Tree('start', ['foo12']),
+            ])
+            self.assertEqual(tree, expected)
+
+            l = Lark(grammar, ambiguity='resolve', lexer=LEXER)
+            tree = l.parse('foo12')
+            self.assertEqual(tree, Tree('start', ['foo1']))
 
         def test_cycle(self):
             grammar = """