Skip to content

Commit 0d63896

Browse files
yoavcloudyoavcloud
authored
Improve accuracy of supports_string_literal_concatenation_with_newline (apache#2348)
1 parent b6af2ae commit 0d63896

7 files changed

Lines changed: 35 additions & 28 deletions

File tree

‎src/ast/comments.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ impl Comments {
7171
/// // all comments appearing before line seven, i.e. before the first statement itself
7272
/// assert_eq!(
7373
/// &comments.find(..Location::new(7, 1)).map(|c| c.as_str()).collect::<Vec<_>>(),
74-
/// &["\n header comment ...\n ... spanning multiple lines\n", " first statement\n"]);
74+
/// &["\n header comment ...\n ... spanning multiple lines\n", " first statement"]);
7575
///
7676
/// // all comments appearing within the first statement
7777
/// assert_eq!(
@@ -81,7 +81,7 @@ impl Comments {
8181
/// // all comments appearing within or after the first statement
8282
/// assert_eq!(
8383
/// &comments.find(Location::new(7, 1)..).map(|c| c.as_str()).collect::<Vec<_>>(),
84-
/// &[" world ", " second statement\n", " trailing comment\n"]);
84+
/// &[" world ", " second statement", " trailing comment"]);
8585
/// ```
8686
///
8787
/// The [Spanned](crate::ast::Spanned) trait allows you to access location

‎src/ast/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12085,7 +12085,8 @@ impl fmt::Display for OptimizerHint {
1208512085
f.write_str(prefix)?;
1208612086
f.write_str(&self.prefix)?;
1208712087
f.write_str("+")?;
12088-
f.write_str(&self.text)
12088+
f.write_str(&self.text)?;
12089+
f.write_str("\n")
1208912090
}
1209012091
OptimizerHintStyle::MultiLine => {
1209112092
f.write_str("/*")?;

‎src/tokenizer.rs‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,7 @@ impl fmt::Display for Whitespace {
521521
Whitespace::Space => f.write_str(" "),
522522
Whitespace::Newline => f.write_str("\n"),
523523
Whitespace::Tab => f.write_str("\t"),
524-
Whitespace::SingleLineComment { prefix, comment } => write!(f, "{prefix}{comment}"),
524+
Whitespace::SingleLineComment { prefix, comment } => writeln!(f, "{prefix}{comment}"),
525525
Whitespace::MultiLineComment(s) => write!(f, "/*{s}*/"),
526526
}
527527
}
@@ -2037,18 +2037,11 @@ impl<'a> Tokenizer<'a> {
20372037

20382038
// Consume characters until newline
20392039
fn tokenize_single_line_comment(&self, chars: &mut State) -> String {
2040-
let mut comment = peeking_take_while(chars, |ch| match ch {
2040+
peeking_take_while(chars, |ch| match ch {
20412041
'\n' => false, // Always stop at \n
20422042
'\r' if dialect_of!(self is PostgreSqlDialect) => false, // Stop at \r for Postgres
20432043
_ => true, // Keep consuming for other characters
2044-
});
2045-
2046-
if let Some(ch) = chars.next() {
2047-
assert!(ch == '\n' || ch == '\r');
2048-
comment.push(ch);
2049-
}
2050-
2051-
comment
2044+
})
20522045
}
20532046

20542047
/// Tokenize an identifier or keyword, after the first char is already consumed.
@@ -3346,8 +3339,9 @@ mod tests {
33463339
Token::Number("0".to_string(), false),
33473340
Token::Whitespace(Whitespace::SingleLineComment {
33483341
prefix: "--".to_string(),
3349-
comment: "this is a comment\n".to_string(),
3342+
comment: "this is a comment".to_string(),
33503343
}),
3344+
Token::Whitespace(Whitespace::Newline),
33513345
Token::Number("1".to_string(), false),
33523346
],
33533347
),
@@ -3367,8 +3361,9 @@ mod tests {
33673361
Token::Number("0".to_string(), false),
33683362
Token::Whitespace(Whitespace::SingleLineComment {
33693363
prefix: "--".to_string(),
3370-
comment: "this is a comment\r\n".to_string(),
3364+
comment: "this is a comment\r".to_string(),
33713365
}),
3366+
Token::Whitespace(Whitespace::Newline),
33723367
Token::Number("1".to_string(), false),
33733368
],
33743369
),
@@ -3392,8 +3387,9 @@ mod tests {
33923387
Token::Number("1".to_string(), false),
33933388
Token::Whitespace(Whitespace::SingleLineComment {
33943389
prefix: "--".to_string(),
3395-
comment: "\r".to_string(),
3390+
comment: "".to_string(),
33963391
}),
3392+
Token::Whitespace(Whitespace::Newline), // Postgres treats \r as newline in single-line comments
33973393
Token::Number("0".to_string(), false),
33983394
];
33993395
compare(expected, tokens);
@@ -4220,16 +4216,19 @@ mod tests {
42204216
vec![
42214217
Token::Whitespace(Whitespace::SingleLineComment {
42224218
prefix: "--".to_string(),
4223-
comment: "\n".to_string(),
4219+
comment: "".to_string(),
42244220
}),
4221+
Token::Whitespace(Whitespace::Newline),
42254222
Token::Whitespace(Whitespace::SingleLineComment {
42264223
prefix: "--".to_string(),
4227-
comment: " Table structure for table...\n".to_string(),
4224+
comment: " Table structure for table...".to_string(),
42284225
}),
4226+
Token::Whitespace(Whitespace::Newline),
42294227
Token::Whitespace(Whitespace::SingleLineComment {
42304228
prefix: "--".to_string(),
4231-
comment: "\n".to_string(),
4229+
comment: "".to_string(),
42324230
}),
4231+
Token::Whitespace(Whitespace::Newline),
42334232
],
42344233
);
42354234
}

‎tests/sqlparser_comments.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,10 @@ more*/
5050
vec![
5151
CommentWithSpan {
5252
comment: Comment::SingleLine {
53-
content: " second line comment\n".into(),
53+
content: " second line comment".into(),
5454
prefix: "--".into()
5555
},
56-
span: Span::new((2, 1).into(), (3, 1).into()),
56+
span: Span::new((2, 1).into(), (2, 23).into()),
5757
},
5858
CommentWithSpan {
5959
comment: Comment::MultiLine(" inline comment after `from` ".into()),

‎tests/sqlparser_common.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18357,6 +18357,14 @@ fn parse_adjacent_string_literal_concatenation() {
1835718357
'd'
1835818358
)"#;
1835918359
dialects.one_statement_parses_to(sql, "SELECT 'abc' IN ('abc', 'd')");
18360+
18361+
let sql = r#"
18362+
SELECT 'abc' in ('a'
18363+
'b' -- COMMENT
18364+
'c',
18365+
'd'
18366+
)"#;
18367+
dialects.one_statement_parses_to(sql, "SELECT 'abc' IN ('abc', 'd')");
1836018368
}
1836118369

1836218370
#[test]

‎tests/sqlparser_oracle.rs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -364,10 +364,7 @@ fn test_optimizer_hints() {
364364
"SELECT --+ one two three /* asdf */\n 1 FROM dual",
365365
);
366366
assert_eq!(select.optimizer_hints.len(), 1);
367-
assert_eq!(
368-
select.optimizer_hints[0].text,
369-
" one two three /* asdf */\n"
370-
);
367+
assert_eq!(select.optimizer_hints[0].text, " one two three /* asdf */");
371368
assert_eq!(select.optimizer_hints[0].prefix, "");
372369

373370
// inserts
@@ -396,7 +393,7 @@ fn test_optimizer_hints() {
396393
);
397394
assert_eq!(select.optimizer_hints.len(), 1);
398395
assert_eq!(select.optimizer_hints[0].prefix, "abc");
399-
assert_eq!(select.optimizer_hints[0].text, " text\n");
396+
assert_eq!(select.optimizer_hints[0].text, " text");
400397
}
401398

402399
#[test]

‎tests/sqlparser_snowflake.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -596,8 +596,9 @@ fn test_snowflake_single_line_tokenize() {
596596
Token::make_keyword("TABLE"),
597597
Token::Whitespace(Whitespace::SingleLineComment {
598598
prefix: "#".to_string(),
599-
comment: " this is a comment \n".to_string(),
599+
comment: " this is a comment ".to_string(),
600600
}),
601+
Token::Whitespace(Whitespace::Newline),
601602
Token::make_word("table_1", None),
602603
];
603604

@@ -613,8 +614,9 @@ fn test_snowflake_single_line_tokenize() {
613614
Token::Whitespace(Whitespace::Space),
614615
Token::Whitespace(Whitespace::SingleLineComment {
615616
prefix: "//".to_string(),
616-
comment: " this is a comment \n".to_string(),
617+
comment: " this is a comment ".to_string(),
617618
}),
619+
Token::Whitespace(Whitespace::Newline),
618620
Token::make_word("table_1", None),
619621
];
620622

0 commit comments

Comments
 (0)