@@ -17,13 +17,13 @@ def self.detect?(text)
1717 return true if text . shebang? 'perl'
1818 end
1919
20- keywords = %w(
20+ KEYWORDS = Set . new %w(
2121 case continue do else elsif for foreach if last my next our
2222 redo reset then unless until while use print new BEGIN CHECK
2323 INIT END return
2424 )
2525
26- builtins = %w(
26+ BUILTINS = Set . new %w(
2727 abs accept alarm atan2 bind binmode bless caller chdir chmod
2828 chomp chop chown chr chroot close closedir connect continue cos
2929 crypt dbmclose dbmopen defined delete die dump each endgrent
@@ -49,60 +49,146 @@ def self.detect?(text)
4949 utime values vec wait waitpid wantarray warn write
5050 )
5151
52+ OPERATOR_WORDS = Set . new %w( eq lt gt le ge ne not and or cmp )
53+
5254 re_tok = Str ::Regex
5355
5456 state :balanced_regex do
55- rule %r(/(\\ [\\ /]|[^/])*/[egimosx]*)m , re_tok , :pop!
56- rule %r(!(\\ [\\ !]|[^!])*![egimosx]*)m , re_tok , :pop!
57- rule %r(\\ (\\ \\ |[^\\ ])*\\ [egimosx]*)m , re_tok , :pop!
58- rule %r({(\\ [\\ }]|[^}])*}[egimosx]*) , re_tok , :pop!
59- rule %r(<(\\ [\\ >]|[^>])*>[egimosx]*) , re_tok , :pop!
60- rule %r(\[ (\\ [\\ \] ]|[^\] ])*\] [egimosx]*) , re_tok , :pop!
61- rule %r[\( (\\ [\\ \) ]|[^\) ])*\) [egimosx]*] , re_tok , :pop!
62- rule %r(@(\\ [\\ @]|[^@])*@[egimosx]*) , re_tok , :pop!
63- rule %r(%(\\ [\\ %]|[^%])*%[egimosx]*) , re_tok , :pop!
64- rule %r(\$ (\\ [\\ \$ ]|[^\$ ])*\$ [egimosx]*) , re_tok , :pop!
57+ rule %r/\s +/ , Text
58+ rule %r/./ do |m |
59+ pop!
60+ open_regex! ( m [ 0 ] )
61+ token Str ::Delimiter
62+ end
6563 end
6664
67- state :root do
65+ state :continued_regex do
66+ mixin :regex
67+ end
68+
69+ state :regex_flags do
70+ rule %r/[msixpodualngcr]+/ , Str ::Affix
71+ rule ( // ) { pop! }
72+ end
73+
74+ state :regex_escapes do
75+ rule %r/\\ [0-7][0-7][0-7]/ , Str ::Escape
76+ rule %r/\\ x\h \h / , Str ::Escape
77+ rule %r/\\ .(?:[{]\w +[}])?/ , Str ::Escape
78+ end
79+
80+ state :regex do
81+ rule %r/./ do |m |
82+ if m [ 0 ] == @regex_end
83+ token Str ::Delimiter
84+ goto :regex_flags
85+ else
86+ fallthrough!
87+ end
88+ end
89+
90+ mixin :regex_escapes
91+ rule %r/[{]\d +(?:,\d +)?[}]/ , Operator
92+ rule %r/\[ \^ ?/ , Punctuation , :regex_char_class
93+ rule %r/[?]|[.]|[|]|[*+][?]?/ , Operator
94+ rule %r/[(](?:[?][=!<:]?)?/ , Punctuation
95+ rule %r/[(][?]<!/ , Punctuation
96+ rule %r/[{})]/ , Punctuation
97+ rule %r/./ , Str ::Regex
98+ end
99+
100+ state :regex_char_class do
101+ rule ( /\^ / ) { token Punctuation ; goto :regex_char_class_inner }
102+ rule ( /-/ ) { token Str ::Regex ; goto :regex_char_class_inner }
103+ rule ( // ) { goto :regex_char_class_inner }
104+ end
105+
106+ state :regex_char_class_inner do
107+ mixin :regex_escapes
108+ rule %r/-(?!\] )/ , Punctuation
109+ rule %r/\\ ./ , Str ::Escape
110+ rule %r/[^-\] \\ ]+/ , Str ::Regex
111+ rule %r/\] / , Punctuation , :pop!
112+ end
113+
114+ BALANCED_DELIMITERS = {
115+ '{' => '}' ,
116+ '(' => ')' ,
117+ '[' => ']' ,
118+ '<' => '>' ,
119+ }
120+
121+ def open_regex! ( delimiter )
122+ @regex_end = BALANCED_DELIMITERS . fetch ( delimiter , delimiter )
123+ push :regex
124+ end
125+
126+ def open_regex_operator! ( delimiter )
127+ if BALANCED_DELIMITERS . key? ( delimiter )
128+ push :balanced_regex
129+ else
130+ push :continued_regex
131+ end
132+
133+ open_regex! ( delimiter )
134+ end
135+
136+ state :expr_start do
137+ mixin :whitespace
138+ rule %r(/) do
139+ open_regex! ( '/' )
140+ token Str ::Delimiter
141+ end
142+
143+ rule ( // ) { pop! }
144+ end
145+
146+ state :whitespace do
68147 rule %r/#.*/ , Comment ::Single
148+ rule %r/\s +/ , Text
69149 rule %r/^=[a-zA-Z0-9]+\s +.*?\n =cut/m , Comment ::Multiline
70- rule %r/(?:#{ keywords . join ( '|' ) } \b / , Keyword
150+ end
151+
152+ state :root do
153+ mixin :whitespace
71154
72155 rule %r/(format)(\s +)([a-zA-Z0-9_]+)(\s *)(=)(\s *\n )/ do
73156 groups Keyword , Text , Name , Text , Punctuation , Text
74157
75158 push :format
76159 end
77160
78- rule %r/(?:eq|lt|gt|le|ge|ne|not|and|or|cmp)\b / , Operator ::Word
79-
80- # substitution/transliteration: balanced delimiters
81- rule %r((?:s|tr|y){(\\ \\ |\\ }|[^}])*}\s *) , re_tok , :balanced_regex
82- rule %r((?:s|tr|y)<(\\ \\ |\\ >|[^>])*>\s *) , re_tok , :balanced_regex
83- rule %r((?:s|tr|y)\[ (\\ \\ |\\ \] |[^\] ])*\] \s *) , re_tok , :balanced_regex
84- rule %r[(?:s|tr|y)\( (\\ \\ |\\ \) |[^\) ])*\) \s *] , re_tok , :balanced_regex
85-
86- # substitution/transliteration: arbitrary non-whitespace delimiters
87- rule %r((?:s|tr|y)\s *([^\w \s ])((\\ \\ |\\ \1 )|[^\1 ])*?\1 ((\\ \\ |\\ \1 )|[^\1 ])*?\1 [msixpodualngcr]*)m , re_tok
88- rule %r((?:s|tr|y)\s +(\w )((\\ \\ |\\ \1 )|[^\1 ])*?\1 ((\\ \\ |\\ \1 )|[^\1 ])*?\1 [msixpodualngcr]*)m , re_tok
89-
90- # matches: common case, m-optional
91- rule %r(m?/(\\ \\ |\\ /|[^/\n ])*/[msixpodualngc]*) , re_tok
92- rule %r(m(?=[/!\\ {<\[ \( @%\$ ])) , re_tok , :balanced_regex
161+ rule %r/\w +/ do |m |
162+ w = m [ 0 ]
163+ if KEYWORDS . include? ( w )
164+ token Keyword
165+ elsif OPERATOR_WORDS . include? ( w )
166+ token Operator ::Word
167+ else
168+ fallthrough!
169+ end
170+ end
93171
94- # arbitrary non-whitespace delimiters
95- rule %r(m \s *([^ \w \s ])(( \\ \\ | \\ \1 )|[^ \1 ])*? \1 [msixpodualngc]*)m , re_tok
96- rule %r(m \s +( \w )(( \\ \\ | \\ \1 )|[^ \1 ])*? \1 [msixpodualngc]*)m , re_tok
172+ rule %r{(?=[a-z_] \w *( \s *#.* \n )* \s *=>)}i do
173+ push :fat_comma
174+ end
97175
98- rule %r(((?<==~)|(?<=\( ))\s */(\\ \\ |\\ /|[^/])*/[msixpodualngc]*) ,
99- re_tok , :balanced_regex
176+ # non-whitespace delimiters that match \w are allowed only
177+ # if there is whitespace between the operator and the delimiter.
178+ # here we use a \b to detect that case in a way that shouldn't
179+ # cause backtracking explosion.
180+ regex_delim = /[^\w \s ]|\b \w /
100181
101- rule %r/\s +/ , Text
182+ rule %r/(s|tr|y)(\s *)(#{ regex_delim } do |m |
183+ open_regex_operator! ( m [ 3 ] )
184+ groups Str ::Affix , Text , Str ::Delimiter
185+ end
102186
103- rule ( /(?=[a-z_]\w *(\s *#.*\n )*\s *=>)/i ) { push :fat_comma }
187+ rule %r/(m)(\s *)(#{ regex_delim } do |m |
188+ open_regex! ( m [ 3 ] )
189+ groups Str ::Affix , Text , Str ::Delimiter
190+ end
104191
105- rule %r/(?:#{ builtins . join ( '|' ) } \b / , Name ::Builtin
106192 rule %r/((__(DIE|WARN)__)|(DATA|STD(IN|OUT|ERR)))\b / ,
107193 Name ::Builtin ::Pseudo
108194
@@ -113,7 +199,9 @@ def self.detect?(text)
113199 rule %r/\$ [\\ "'\[ \] &`+*.,;=%~?@$!<>(^\| \/ _-](?!\w )/ , Name ::Variable ::Global
114200 rule %r/[$@%&*][$@%&*#_]*(?=[a-z{\[ ;])/i , Name ::Variable , :varname
115201
116- rule %r/[-+\/ *%=<>&^\| !\\ ~]=?/ , Operator
202+ rule %r/\[ \] |\* \* |::|<<|>>|>=|<=|<=>|={3}|!=|=~|!~|&&?|\| \| |\. {1,3}/ ,
203+ Operator , :expr_start
204+ rule %r/[-+\/ *%=<>&^\| !\\ ~]=?/ , Operator , :expr_start
117205
118206 rule %r/0_?[0-7]+(_[0-7]+)*/ , Num ::Oct
119207 rule %r/0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*/ , Num ::Hex
@@ -133,11 +221,20 @@ def self.detect?(text)
133221 rule %r/(q|qq|qw|qr|qx)</ , Str ::Other , :lt_string
134222 rule %r/(q|qq|qw|qr|qx)(\W )(.|\n )*?\2 / , Str ::Other
135223
136- rule %r/package\s +/ , Keyword , :modulename
137- rule %r/sub\s +/ , Keyword , :funcname
138- rule %r/\[ \] |\* \* |::|<<|>>|>=|<=|<=>|={3}|!=|=~|!~|&&?|\| \| |\. {1,3}/ ,
139- Operator
140- rule %r/[()\[ \] :;,<>\/ ?{}]/ , Punctuation
224+ rule %r/package\b / , Keyword , :modulename
225+ rule %r/sub\b / , Keyword , :funcname
226+ rule %r/[(]/ , Punctuation , :expr_start
227+ rule %r/[)\[ \] :;,<>\/ ?{}]/ , Punctuation
228+
229+ rule %r/[a-z]\w */ do |m |
230+ w = m [ 0 ]
231+ if BUILTINS . include? ( w )
232+ token Name ::Builtin
233+ else
234+ fallthrough!
235+ end
236+ end
237+
141238 rule ( /(?=\w )/ ) { push :name }
142239 end
143240
0 commit comments