Handle comments before rule colons

Author: ydah

lib/lrama/lexer.rb

@@ -62,6 +62,7 @@ class Lexer
       %categories
       %start
     ).freeze #: Array[String]
+    IDENTIFIER_PATTERN = /[a-zA-Z_.][-a-zA-Z0-9_.]*/.freeze #: Regexp
 
     # @rbs (GrammarFile grammar_file) -> void
     def initialize(grammar_file)
@@ -135,10 +136,10 @@ def lex_token
         return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)]
       when @scanner.scan(/\d+/)
         return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)]
-      when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/)
+      when @scanner.scan(IDENTIFIER_PATTERN)
         token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location)
         type =
-          if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/)
+          if identifier_colon?
             :IDENT_COLON
           else
             :IDENTIFIER
@@ -196,6 +197,36 @@ def lex_c_code
 
     private
 
+    # @rbs () -> bool
+    def identifier_colon?
+      scanner = StringScanner.new(@scanner.rest)
+      skip_trivia(scanner)
+
+      if scanner.scan(/\[/)
+        skip_trivia(scanner)
+        return false unless scanner.scan(IDENTIFIER_PATTERN)
+
+        skip_trivia(scanner)
+        return false unless scanner.scan(/\]/)
+      end
+
+      skip_trivia(scanner)
+      !scanner.scan(/:/).nil?
+    end
+
+    # @rbs (StringScanner scanner) -> void
+    def skip_trivia(scanner)
+      loop do
+        case
+        when scanner.scan(/\s+/)
+        when scanner.scan(/\/\*[\s\S]*?\*\//)
+        when scanner.scan(%r{//.*(?:\n|$)})
+        else
+          return
+        end
+      end
+    end
+
     # @rbs () -> void
     def lex_comment
       until @scanner.eos? do

sig/generated/lrama/lexer.rbs

@@ -452,4 +452,18 @@
       }
     CODE
   end
+
+  it 'lexes a rule name with a block comment before the colon as IDENT_COLON' do
+    grammar_file = Lrama::Lexer::GrammarFile.new("commented_rule_lhs.y", <<~INPUT)
+      %%
+      stmt
+        /* some block comment */
+        :
+    INPUT
+    lexer = Lrama::Lexer.new(grammar_file)
+
+    expect(lexer.next_token).to eq(['%%', token_class::Token.new(s_value: '%%')])
+    expect(lexer.next_token).to eq([:IDENT_COLON, token_class::Ident.new(s_value: 'stmt')])
+    expect(lexer.next_token).to eq([':', token_class::Token.new(s_value: ':')])
+  end
 end

spec/lrama/lexer_spec.rb

@@ -4323,6 +4323,25 @@ class : keyword_class tSTRING keyword_end { code 1 }
           end
         end
 
+        it "can parse a rule when a block comment appears before the colon" do
+          y = <<~INPUT
+            %%
+
+            program:
+                stmt
+            ;
+
+            stmt
+                /* some block comment */
+                : %empty
+            ;
+          INPUT
+
+          parser = Lrama::Parser.new(y, "parse.y")
+
+          expect { parser.parse }.not_to raise_error
+        end
+
         context "includes invalid named references" do
           it "raise an error" do
             y = <<~INPUT