PostgreSQL Tokenization: Fix unexpected characters after question mark being silently ignored (apache#2129)

Author: jnlt3

src/tokenizer.rs

@@ -1795,7 +1795,7 @@ impl<'a> Tokenizer<'a> {
                             }
                         }
                         Some('#') => self.consume_and_return(chars, Token::QuestionMarkSharp),
-                        _ => self.consume_and_return(chars, Token::Question),
+                        _ => Ok(Some(Token::Question)),
                     }
                 }
                 '?' => {
@@ -4240,4 +4240,23 @@ mod tests {
             ],
         );
     }
+
+    #[test]
+    fn tokenize_question_mark() {
+        let dialect = PostgreSqlDialect {};
+        let sql = "SELECT x ? y";
+        let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
+        compare(
+            tokens,
+            vec![
+                Token::make_keyword("SELECT"),
+                Token::Whitespace(Whitespace::Space),
+                Token::make_word("x", None),
+                Token::Whitespace(Whitespace::Space),
+                Token::Question,
+                Token::Whitespace(Whitespace::Space),
+                Token::make_word("y", None),
+            ],
+        )
+    }
 }