Fix parsing xpath that has string literal containing parentheses and brackets (#318)

String literal in XPath 1.0 doesn't have escapes.
`begin end while cond` is not a normal modifier while, but a do-while
loop. (I didn't know it). REXML uses this syntax in several files.

Removes preprocessing that removes spaces with gsub which may wrongly
modifies string literal.
Also fixes error parsing `//a[1] [2]`.
All test except exact-error-message test and the one I added in this PR
passed with:
```ruby
def parse path
  path = path.gsub('(', '( ').gsub(/[\)\[\]]/, ' \0 ') # Insert space around paren/bracket
```
REF: https://www.w3.org/TR/xpath-10/#exprlex
> For readability, whitespace may be used in expressions even though not
explicitly allowed by the grammar: ExprWhitespace may be freely added
within patterns before or after any ExprToken.

Author: tompng

lib/rexml/parsers/xpathparser.rb

@@ -19,9 +19,6 @@ def namespaces=( namespaces )
       end
 
       def parse path
-        path = path.dup
-        path.gsub!(/([\(\[])\s+/, '\1') # Strip ignorable spaces
-        path.gsub!( /\s+([\]\)])/, '\1')
         parsed = []
         rest = OrExpr(path, parsed)
         if rest
@@ -359,17 +356,15 @@ def NodeTest path, parsed
           path = $'
           parsed << type.tr('-', '_').intern
         when PI
-          path = $'
+          path = $'.lstrip
           literal = nil
-          if path =~ /^\s*\)/
-            path = $'
-          else
+          unless path.start_with?(')')
             path =~ LITERAL
             literal = $1
-            path = $'
+            path = $'.lstrip
             raise ParseException.new("Missing ')' after processing instruction") if path[0] != ?)
-            path = path[1..-1]
           end
+          path = path[1..-1]
           parsed << :processing_instruction
           parsed << (literal || '')
         when LOCAL_NAME_WILDCARD
@@ -400,6 +395,7 @@ def Predicate path, parsed
         while path[0] == ?[
           path, expr = get_group(path)
           predicates << expr[1..-2] if expr
+          path = path.lstrip
         end
         predicates.each{ |pred|
           preds = []
@@ -678,12 +674,20 @@ def get_group string
         depth = 0
         st = string[0,1]
         en = (st == "(" ? ")" : "]")
+        quote = nil
         begin
-          case string[ind,1]
-          when st
-            depth += 1
-          when en
-            depth -= 1
+          if quote
+            # ignore () [] inside quotes
+            quote = nil if string[ind] == quote
+          else
+            case string[ind]
+            when st
+              depth += 1
+            when en
+              depth -= 1
+            when '"', "'"
+              quote = string[ind]
+            end
           end
           ind += 1
         end while depth > 0 and ind < string.length

test/parser/test_xpath.rb

@@ -111,5 +111,23 @@ def test_filter_string_double_quote
                      abbreviate("a/b[attribute::name='double \" quote']/c"))
       end
     end
+
+    def test_spaces_between_tokens
+      # REXML doesn't support space between function-name and opening paren.
+      # Spaces after `(` and spaces around `)`, `[`, `]` are tested here.
+      parser = REXML::Parsers::XPathParser.new
+      assert_equal(
+        parser.parse('//a/b[c][d]/e'),
+        parser.parse(' // a / b [ c ] [ d ] / e '),
+      )
+      assert_equal(
+        parser.parse('/a/b[string-length("1")<(2+3)]/c'),
+        parser.parse(' / a / b [ string-length( "1" ) < ( 2 + 3 ) ] / c '),
+      )
+      assert_equal(
+        parser.parse('//processing-instruction("a")'),
+        parser.parse('//processing-instruction( "a" )'),
+      )
+    end
   end
 end

test/xpath/test_base.rb

@@ -830,6 +830,24 @@ def test_spaces
                    match.call('/ a / child:: c [( @id )] /'))
     end
 
+    def test_space_paren_brace_inside_xpath_string
+      doc = Document.new(<<~XML)
+        <a>
+          <b id=" [ ' 1 ) "/>
+          <b id=' ( " 2 ] '/>
+        </a>
+      XML
+
+      assert_equal(
+        [" [ ' 1 ) "],
+        REXML::XPath.match(doc, "/a/b[@id=\" [ ' 1 ) \"]").map { |e| e.attributes['id'] }
+      )
+      assert_equal(
+        [' ( " 2 ] '],
+        REXML::XPath.match(doc, "/a/b[@id=' ( \" 2 ] ']").map { |e| e.attributes['id'] }
+      )
+    end
+
     def test_text_nodes
       #  source = "<root>
       #<child/>