Skip to content

Commit cb25e96

Browse files
committed
Tokenize empty line comments correctly
MySQL likes to send "blocky" comments where a line comment is surrounded by empty comments, like so: ``` -- -- Table structure for table `foo` -- CREATE TABLE ... ``` Currently, these comments are tokenized as MINUS, MINUS instead of an empty line comment. This commits fixes this problem by accepting any kind of whitespace to delimit a line comment.
1 parent 3c61db5 commit cb25e96

File tree

1 file changed

+23
-1
lines changed

1 file changed

+23
-1
lines changed

src/tokenizer.rs

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1326,7 +1326,11 @@ impl<'a> Tokenizer<'a> {
13261326
Some('-') => {
13271327
let mut is_comment = true;
13281328
if self.dialect.requires_single_line_comment_whitespace() {
1329-
is_comment = Some(' ') == chars.peekable.clone().nth(1);
1329+
is_comment = chars
1330+
.peekable
1331+
.clone()
1332+
.nth(1)
1333+
.is_some_and(char::is_whitespace);
13301334
}
13311335

13321336
if is_comment {
@@ -3964,6 +3968,24 @@ mod tests {
39643968
Token::Minus,
39653969
],
39663970
);
3971+
3972+
all_dialects_where(|d| d.requires_single_line_comment_whitespace()).tokenizes_to(
3973+
"--\n-- Table structure for table...\n--\n",
3974+
vec![
3975+
Token::Whitespace(Whitespace::SingleLineComment {
3976+
prefix: "--".to_string(),
3977+
comment: "\n".to_string(),
3978+
}),
3979+
Token::Whitespace(Whitespace::SingleLineComment {
3980+
prefix: "--".to_string(),
3981+
comment: " Table structure for table...\n".to_string(),
3982+
}),
3983+
Token::Whitespace(Whitespace::SingleLineComment {
3984+
prefix: "--".to_string(),
3985+
comment: "\n".to_string(),
3986+
}),
3987+
],
3988+
);
39673989
}
39683990

39693991
#[test]

0 commit comments

Comments
 (0)