parser: validate string continuations (#1137)

Author: sbdchd

crates/squawk_ide/src/infer.rs

@@ -131,6 +131,11 @@ mod tests {
         assert_snapshot!(infer("select E'hello'"), @"text");
     }
 
+    #[test]
+    fn unicode_escape_string() {
+        assert_snapshot!(infer("select U&' \' UESCAPE '!'"), @"text");
+    }
+
     #[test]
     fn boolean_true() {
         assert_snapshot!(infer("select true"), @"boolean");
@@ -161,6 +166,11 @@ mod tests {
         assert_snapshot!(infer("select b'100'"), @"bit");
     }
 
+    #[test]
+    fn byte_string() {
+        assert_snapshot!(infer("select x'FF'"), @"bit");
+    }
+
     #[test]
     fn bit_varying() {
         assert_snapshot!(infer("select b'100'::bit varying"), @"bit");

crates/squawk_parser/src/lexed_str.rs

@@ -255,25 +255,17 @@ impl<'a> Converter<'a> {
                         "Missing trailing `'` symbol to terminate the hex bit string literal"
                             .into(),
                     );
-                } else {
-                    let inside = &token_text[2..token_text.len() - 1];
-                    if let Some(c) = inside.chars().find(|c| !c.is_ascii_hexdigit()) {
-                        err = Some(format!("\"{c}\" is not a valid hexadecimal digit"));
-                    }
                 }
+                // digit validation in squawk_syntax
                 SyntaxKind::BYTE_STRING
             }
             squawk_lexer::LiteralKind::BitStr { terminated } => {
                 if !terminated {
                     err = Some(
                         "Missing trailing `'` symbol to terminate the bit string literal".into(),
                     );
-                } else {
-                    let inside = &token_text[2..token_text.len() - 1];
-                    if let Some(c) = inside.chars().find(|&c| c != '0' && c != '1') {
-                        err = Some(format!("\"{c}\" is not a valid binary digit"));
-                    }
                 }
+                // digit validation in squawk_syntax
                 SyntaxKind::BIT_STRING
             }
             squawk_lexer::LiteralKind::DollarQuotedString { terminated } => {
@@ -298,9 +290,8 @@ impl<'a> Converter<'a> {
                     err = Some(
                         "Missing trailing `'` symbol to terminate the escape string literal".into(),
                     );
-                } else {
-                    err = validate_escape_string_unicode_escapes(token_text);
                 }
+                // unicode escape sequences validated in squawk_syntax
                 SyntaxKind::ESC_STRING
             }
         };
@@ -309,32 +300,6 @@ impl<'a> Converter<'a> {
     }
 }
 
-fn validate_escape_string_unicode_escapes(token_text: &str) -> Option<String> {
-    let mut chars = token_text[2..token_text.len() - 1].chars();
-
-    while let Some(c) = chars.next() {
-        if c != '\\' {
-            continue;
-        }
-
-        let (required, example) = match chars.next() {
-            Some('u') => (4, r"\uXXXX"),
-            Some('U') => (8, r"\UXXXXXXXX"),
-            _ => continue,
-        };
-
-        for _ in 0..required {
-            if !chars.next().is_some_and(|c| c.is_ascii_hexdigit()) {
-                return Some(format!(
-                    "Unicode escape requires {required} hex digits: {example}"
-                ));
-            }
-        }
-    }
-
-    None
-}
-
 #[cfg(test)]
 mod tests {
     use annotate_snippets::{AnnotationKind, Level, Renderer, Snippet, renderer::DecorStyle};
@@ -390,16 +355,6 @@ mod tests {
         ");
     }
 
-    #[test]
-    fn hex_invalid_digit() {
-        assert_snapshot!(lex("select X'1FZ';"), @r#"
-        error: "Z" is not a valid hexadecimal digit
-          ╭▸ 
-        1 │ select X'1FZ';
-          ╰╴       ━━━━━━
-        "#);
-    }
-
     #[test]
     fn unterminated_hex_bit_string_error() {
         assert_snapshot!(lex("select X'1F;"), @"
@@ -420,16 +375,6 @@ mod tests {
         ");
     }
 
-    #[test]
-    fn invalid_binary_digit_error() {
-        assert_snapshot!(lex("select b'0 ';"), @r#"
-        error: " " is not a valid binary digit
-          ╭▸ 
-        1 │ select b'0 ';
-          ╰╴       ━━━━━
-        "#);
-    }
-
     #[test]
     fn unterminated_dollar_quoted_string_error() {
         assert_snapshot!(lex("select $tag$hello;"), @"
@@ -459,24 +404,4 @@ mod tests {
           ╰╴       ━━━━━━━━
         ");
     }
-
-    #[test]
-    fn invalid_unicode_escape_4_digits_error() {
-        assert_snapshot!(lex(r"select E'\u00';"), @r"
-        error: Unicode escape requires 4 hex digits: \uXXXX
-          ╭▸ 
-        1 │ select E'\u00';
-          ╰╴       ━━━━━━━
-        ");
-    }
-
-    #[test]
-    fn invalid_unicode_escape_8_digits_error() {
-        assert_snapshot!(lex(r"select E'\UFFFF';"), @r"
-        error: Unicode escape requires 8 hex digits: \UXXXXXXXX
-          ╭▸ 
-        1 │ select E'\UFFFF';
-          ╰╴       ━━━━━━━━━
-        ");
-    }
 }

crates/squawk_syntax/src/snapshots/squawk_syntax__test__bit_string_validation.snap

@@ -0,0 +1,108 @@
+---
+source: crates/squawk_syntax/src/test.rs
+input_file: crates/squawk_syntax/test_data/validation/bit_string.sql
+---
+SOURCE_FILE@0..120
+  COMMENT@0..5 "-- ok"
+  WHITESPACE@5..6 "\n"
+  SELECT@6..18
+    SELECT_CLAUSE@6..18
+      SELECT_KW@6..12 "select"
+      WHITESPACE@12..13 " "
+      TARGET_LIST@13..18
+        TARGET@13..18
+          LITERAL@13..18
+            BIT_STRING@13..18 "b'01'"
+  SEMICOLON@18..19 ";"
+  WHITESPACE@19..20 "\n"
+  SELECT@20..34
+    SELECT_CLAUSE@20..34
+      SELECT_KW@20..26 "select"
+      WHITESPACE@26..27 " "
+      TARGET_LIST@27..34
+        TARGET@27..34
+          LITERAL@27..34
+            BIT_STRING@27..34 "B'1010'"
+  SEMICOLON@34..35 ";"
+  WHITESPACE@35..36 "\n"
+  SELECT@36..46
+    SELECT_CLAUSE@36..46
+      SELECT_KW@36..42 "select"
+      WHITESPACE@42..43 " "
+      TARGET_LIST@43..46
+        TARGET@43..46
+          LITERAL@43..46
+            BIT_STRING@43..46 "b''"
+  SEMICOLON@46..47 ";"
+  WHITESPACE@47..49 "\n\n"
+  COMMENT@49..58 "-- errors"
+  WHITESPACE@58..59 "\n"
+  SELECT@59..72
+    SELECT_CLAUSE@59..72
+      SELECT_KW@59..65 "select"
+      WHITESPACE@65..66 " "
+      TARGET_LIST@66..72
+        TARGET@66..72
+          LITERAL@66..72
+            BIT_STRING@66..72 "b'012'"
+  SEMICOLON@72..73 ";"
+  WHITESPACE@73..74 "\n"
+  SELECT@74..87
+    SELECT_CLAUSE@74..87
+      SELECT_KW@74..80 "select"
+      WHITESPACE@80..81 " "
+      TARGET_LIST@81..87
+        TARGET@81..87
+          LITERAL@81..87
+            BIT_STRING@81..87 "b'01A'"
+  SEMICOLON@87..88 ";"
+  WHITESPACE@88..89 "\n"
+  SELECT@89..102
+    SELECT_CLAUSE@89..102
+      SELECT_KW@89..95 "select"
+      WHITESPACE@95..96 " "
+      TARGET_LIST@96..102
+        TARGET@96..102
+          LITERAL@96..102
+            BIT_STRING@96..102 "b'0 1'"
+  SEMICOLON@102..103 ";"
+  WHITESPACE@103..104 "\n"
+  SELECT@104..118
+    SELECT_CLAUSE@104..118
+      SELECT_KW@104..110 "select"
+      WHITESPACE@110..111 " "
+      TARGET_LIST@111..118
+        TARGET@111..118
+          LITERAL@111..118
+            BIT_STRING@111..118 "B'2345'"
+  SEMICOLON@118..119 ";"
+  WHITESPACE@119..120 "\n"
+
+error[syntax-error]: "2" is not a valid binary digit
+  ╭▸ 
+7 │ select b'012';
+  ╰╴           ━
+error[syntax-error]: "A" is not a valid binary digit
+  ╭▸ 
+8 │ select b'01A';
+  ╰╴           ━
+error[syntax-error]: " " is not a valid binary digit
+  ╭▸ 
+9 │ select b'0 1';
+  ╰╴          ━
+error[syntax-error]: "2" is not a valid binary digit
+   ╭▸ 
+10 │ select B'2345';
+   ╰╴         ━
+error[syntax-error]: "3" is not a valid binary digit
+   ╭▸ 
+10 │ select B'2345';
+   ╰╴          ━
+error[syntax-error]: "4" is not a valid binary digit
+   ╭▸ 
+10 │ select B'2345';
+   ╰╴           ━
+error[syntax-error]: "5" is not a valid binary digit
+   ╭▸ 
+10 │ select B'2345';
+   ╰╴            ━

crates/squawk_syntax/src/snapshots/squawk_syntax__test__byte_string_validation.snap

@@ -0,0 +1,112 @@
+---
+source: crates/squawk_syntax/src/test.rs
+input_file: crates/squawk_syntax/test_data/validation/byte_string.sql
+---
+SOURCE_FILE@0..124
+  COMMENT@0..5 "-- ok"
+  WHITESPACE@5..6 "\n"
+  SELECT@6..18
+    SELECT_CLAUSE@6..18
+      SELECT_KW@6..12 "select"
+      WHITESPACE@12..13 " "
+      TARGET_LIST@13..18
+        TARGET@13..18
+          LITERAL@13..18
+            BYTE_STRING@13..18 "x'1F'"
+  SEMICOLON@18..19 ";"
+  WHITESPACE@19..20 "\n"
+  SELECT@20..38
+    SELECT_CLAUSE@20..38
+      SELECT_KW@20..26 "select"
+      WHITESPACE@26..27 " "
+      TARGET_LIST@27..38
+        TARGET@27..38
+          LITERAL@27..38
+            BYTE_STRING@27..38 "X'deadBEEF'"
+  SEMICOLON@38..39 ";"
+  WHITESPACE@39..40 "\n"
+  SELECT@40..50
+    SELECT_CLAUSE@40..50
+      SELECT_KW@40..46 "select"
+      WHITESPACE@46..47 " "
+      TARGET_LIST@47..50
+        TARGET@47..50
+          LITERAL@47..50
+            BYTE_STRING@47..50 "x''"
+  SEMICOLON@50..51 ";"
+  WHITESPACE@51..53 "\n\n"
+  COMMENT@53..62 "-- errors"
+  WHITESPACE@62..63 "\n"
+  SELECT@63..76
+    SELECT_CLAUSE@63..76
+      SELECT_KW@63..69 "select"
+      WHITESPACE@69..70 " "
+      TARGET_LIST@70..76
+        TARGET@70..76
+          LITERAL@70..76
+            BYTE_STRING@70..76 "x'1FZ'"
+  SEMICOLON@76..77 ";"
+  WHITESPACE@77..78 "\n"
+  SELECT@78..90
+    SELECT_CLAUSE@78..90
+      SELECT_KW@78..84 "select"
+      WHITESPACE@84..85 " "
+      TARGET_LIST@85..90
+        TARGET@85..90
+          LITERAL@85..90
+            BYTE_STRING@85..90 "x'1G'"
+  SEMICOLON@90..91 ";"
+  WHITESPACE@91..92 "\n"
+  SELECT@92..106
+    SELECT_CLAUSE@92..106
+      SELECT_KW@92..98 "select"
+      WHITESPACE@98..99 " "
+      TARGET_LIST@99..106
+        TARGET@99..106
+          LITERAL@99..106
+            BYTE_STRING@99..106 "x'1G2H'"
+  SEMICOLON@106..107 ";"
+  WHITESPACE@107..108 "\n"
+  SELECT@108..122
+    SELECT_CLAUSE@108..122
+      SELECT_KW@108..114 "select"
+      WHITESPACE@114..115 " "
+      TARGET_LIST@115..122
+        TARGET@115..122
+          LITERAL@115..122
+            BYTE_STRING@115..122 "X'GHIJ'"
+  SEMICOLON@122..123 ";"
+  WHITESPACE@123..124 "\n"
+
+error[syntax-error]: "Z" is not a valid hexadecimal digit
+  ╭▸ 
+7 │ select x'1FZ';
+  ╰╴           ━
+error[syntax-error]: "G" is not a valid hexadecimal digit
+  ╭▸ 
+8 │ select x'1G';
+  ╰╴          ━
+error[syntax-error]: "G" is not a valid hexadecimal digit
+  ╭▸ 
+9 │ select x'1G2H';
+  ╰╴          ━
+error[syntax-error]: "H" is not a valid hexadecimal digit
+  ╭▸ 
+9 │ select x'1G2H';
+  ╰╴            ━
+error[syntax-error]: "G" is not a valid hexadecimal digit
+   ╭▸ 
+10 │ select X'GHIJ';
+   ╰╴         ━
+error[syntax-error]: "H" is not a valid hexadecimal digit
+   ╭▸ 
+10 │ select X'GHIJ';
+   ╰╴          ━
+error[syntax-error]: "I" is not a valid hexadecimal digit
+   ╭▸ 
+10 │ select X'GHIJ';
+   ╰╴           ━
+error[syntax-error]: "J" is not a valid hexadecimal digit
+   ╭▸ 
+10 │ select X'GHIJ';
+   ╰╴            ━