Skip to content

parser: validate string continuations #1137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kodiakhq merged 5 commits into from
May 13, 2026
Merged

parser: validate string continuations #1137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
531907b
update snap baseline
sbdchd May 13, 2026
d6c23d3
move to syntax from parser
sbdchd May 13, 2026
7250477
support continuations
sbdchd May 13, 2026
2520bc2
add some tests
sbdchd May 13, 2026
9a9c8fa
fix lint
sbdchd May 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions crates/squawk_ide/src/infer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,11 @@ mod tests {
assert_snapshot!(infer("select E'hello'"), @"text");
}

#[test]
fn unicode_escape_string() {
assert_snapshot!(infer("select U&' \' UESCAPE '!'"), @"text");
}

#[test]
fn boolean_true() {
assert_snapshot!(infer("select true"), @"boolean");
Expand Down Expand Up @@ -161,6 +166,11 @@ mod tests {
assert_snapshot!(infer("select b'100'"), @"bit");
}

#[test]
fn byte_string() {
assert_snapshot!(infer("select x'FF'"), @"bit");
}

#[test]
fn bit_varying() {
assert_snapshot!(infer("select b'100'::bit varying"), @"bit");
Expand Down
81 changes: 3 additions & 78 deletions crates/squawk_parser/src/lexed_str.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,25 +255,17 @@ impl<'a> Converter<'a> {
"Missing trailing `'` symbol to terminate the hex bit string literal"
.into(),
);
} else {
let inside = &token_text[2..token_text.len() - 1];
if let Some(c) = inside.chars().find(|c| !c.is_ascii_hexdigit()) {
err = Some(format!("\"{c}\" is not a valid hexadecimal digit"));
}
}
// digit validation in squawk_syntax
SyntaxKind::BYTE_STRING
}
squawk_lexer::LiteralKind::BitStr { terminated } => {
if !terminated {
err = Some(
"Missing trailing `'` symbol to terminate the bit string literal".into(),
);
} else {
let inside = &token_text[2..token_text.len() - 1];
if let Some(c) = inside.chars().find(|&c| c != '0' && c != '1') {
err = Some(format!("\"{c}\" is not a valid binary digit"));
}
}
// digit validation in squawk_syntax
SyntaxKind::BIT_STRING
}
squawk_lexer::LiteralKind::DollarQuotedString { terminated } => {
Expand All @@ -298,9 +290,8 @@ impl<'a> Converter<'a> {
err = Some(
"Missing trailing `'` symbol to terminate the escape string literal".into(),
);
} else {
err = validate_escape_string_unicode_escapes(token_text);
}
// unicode escape sequences validated in squawk_syntax
SyntaxKind::ESC_STRING
}
};
Expand All @@ -309,32 +300,6 @@ impl<'a> Converter<'a> {
}
}

fn validate_escape_string_unicode_escapes(token_text: &str) -> Option<String> {
let mut chars = token_text[2..token_text.len() - 1].chars();

while let Some(c) = chars.next() {
if c != '\\' {
continue;
}

let (required, example) = match chars.next() {
Some('u') => (4, r"\uXXXX"),
Some('U') => (8, r"\UXXXXXXXX"),
_ => continue,
};

for _ in 0..required {
if !chars.next().is_some_and(|c| c.is_ascii_hexdigit()) {
return Some(format!(
"Unicode escape requires {required} hex digits: {example}"
));
}
}
}

None
}

#[cfg(test)]
mod tests {
use annotate_snippets::{AnnotationKind, Level, Renderer, Snippet, renderer::DecorStyle};
Expand Down Expand Up @@ -390,16 +355,6 @@ mod tests {
");
}

#[test]
fn hex_invalid_digit() {
assert_snapshot!(lex("select X'1FZ';"), @r#"
error: "Z" is not a valid hexadecimal digit
╭▸
1 │ select X'1FZ';
╰╴ ━━━━━━
"#);
}

#[test]
fn unterminated_hex_bit_string_error() {
assert_snapshot!(lex("select X'1F;"), @"
Expand All @@ -420,16 +375,6 @@ mod tests {
");
}

#[test]
fn invalid_binary_digit_error() {
assert_snapshot!(lex("select b'0 ';"), @r#"
error: " " is not a valid binary digit
╭▸
1 │ select b'0 ';
╰╴ ━━━━━
"#);
}

#[test]
fn unterminated_dollar_quoted_string_error() {
assert_snapshot!(lex("select $tag$hello;"), @"
Expand Down Expand Up @@ -459,24 +404,4 @@ mod tests {
╰╴ ━━━━━━━━
");
}

#[test]
fn invalid_unicode_escape_4_digits_error() {
assert_snapshot!(lex(r"select E'\u00';"), @r"
error: Unicode escape requires 4 hex digits: \uXXXX
╭▸
1 │ select E'\u00';
╰╴ ━━━━━━━
");
}

#[test]
fn invalid_unicode_escape_8_digits_error() {
assert_snapshot!(lex(r"select E'\UFFFF';"), @r"
error: Unicode escape requires 8 hex digits: \UXXXXXXXX
╭▸
1 │ select E'\UFFFF';
╰╴ ━━━━━━━━━
");
}
}
108 changes: 108 additions & 0 deletions crates/squawk_syntax/src/snapshots/squawk_syntax__test__bit_string_validation.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
---
source: crates/squawk_syntax/src/test.rs
input_file: crates/squawk_syntax/test_data/validation/bit_string.sql
---
SOURCE_FILE@0..120
COMMENT@0..5 "-- ok"
WHITESPACE@5..6 "\n"
SELECT@6..18
SELECT_CLAUSE@6..18
SELECT_KW@6..12 "select"
WHITESPACE@12..13 " "
TARGET_LIST@13..18
TARGET@13..18
LITERAL@13..18
BIT_STRING@13..18 "b'01'"
SEMICOLON@18..19 ";"
WHITESPACE@19..20 "\n"
SELECT@20..34
SELECT_CLAUSE@20..34
SELECT_KW@20..26 "select"
WHITESPACE@26..27 " "
TARGET_LIST@27..34
TARGET@27..34
LITERAL@27..34
BIT_STRING@27..34 "B'1010'"
SEMICOLON@34..35 ";"
WHITESPACE@35..36 "\n"
SELECT@36..46
SELECT_CLAUSE@36..46
SELECT_KW@36..42 "select"
WHITESPACE@42..43 " "
TARGET_LIST@43..46
TARGET@43..46
LITERAL@43..46
BIT_STRING@43..46 "b''"
SEMICOLON@46..47 ";"
WHITESPACE@47..49 "\n\n"
COMMENT@49..58 "-- errors"
WHITESPACE@58..59 "\n"
SELECT@59..72
SELECT_CLAUSE@59..72
SELECT_KW@59..65 "select"
WHITESPACE@65..66 " "
TARGET_LIST@66..72
TARGET@66..72
LITERAL@66..72
BIT_STRING@66..72 "b'012'"
SEMICOLON@72..73 ";"
WHITESPACE@73..74 "\n"
SELECT@74..87
SELECT_CLAUSE@74..87
SELECT_KW@74..80 "select"
WHITESPACE@80..81 " "
TARGET_LIST@81..87
TARGET@81..87
LITERAL@81..87
BIT_STRING@81..87 "b'01A'"
SEMICOLON@87..88 ";"
WHITESPACE@88..89 "\n"
SELECT@89..102
SELECT_CLAUSE@89..102
SELECT_KW@89..95 "select"
WHITESPACE@95..96 " "
TARGET_LIST@96..102
TARGET@96..102
LITERAL@96..102
BIT_STRING@96..102 "b'0 1'"
SEMICOLON@102..103 ";"
WHITESPACE@103..104 "\n"
SELECT@104..118
SELECT_CLAUSE@104..118
SELECT_KW@104..110 "select"
WHITESPACE@110..111 " "
TARGET_LIST@111..118
TARGET@111..118
LITERAL@111..118
BIT_STRING@111..118 "B'2345'"
SEMICOLON@118..119 ";"
WHITESPACE@119..120 "\n"

error[syntax-error]: "2" is not a valid binary digit
╭▸
7 │ select b'012';
╰╴ ━
error[syntax-error]: "A" is not a valid binary digit
╭▸
8 │ select b'01A';
╰╴ ━
error[syntax-error]: " " is not a valid binary digit
╭▸
9 │ select b'0 1';
╰╴ ━
error[syntax-error]: "2" is not a valid binary digit
╭▸
10 │ select B'2345';
╰╴ ━
error[syntax-error]: "3" is not a valid binary digit
╭▸
10 │ select B'2345';
╰╴ ━
error[syntax-error]: "4" is not a valid binary digit
╭▸
10 │ select B'2345';
╰╴ ━
error[syntax-error]: "5" is not a valid binary digit
╭▸
10 │ select B'2345';
╰╴ ━
112 changes: 112 additions & 0 deletions crates/squawk_syntax/src/snapshots/squawk_syntax__test__byte_string_validation.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
---
source: crates/squawk_syntax/src/test.rs
input_file: crates/squawk_syntax/test_data/validation/byte_string.sql
---
SOURCE_FILE@0..124
COMMENT@0..5 "-- ok"
WHITESPACE@5..6 "\n"
SELECT@6..18
SELECT_CLAUSE@6..18
SELECT_KW@6..12 "select"
WHITESPACE@12..13 " "
TARGET_LIST@13..18
TARGET@13..18
LITERAL@13..18
BYTE_STRING@13..18 "x'1F'"
SEMICOLON@18..19 ";"
WHITESPACE@19..20 "\n"
SELECT@20..38
SELECT_CLAUSE@20..38
SELECT_KW@20..26 "select"
WHITESPACE@26..27 " "
TARGET_LIST@27..38
TARGET@27..38
LITERAL@27..38
BYTE_STRING@27..38 "X'deadBEEF'"
SEMICOLON@38..39 ";"
WHITESPACE@39..40 "\n"
SELECT@40..50
SELECT_CLAUSE@40..50
SELECT_KW@40..46 "select"
WHITESPACE@46..47 " "
TARGET_LIST@47..50
TARGET@47..50
LITERAL@47..50
BYTE_STRING@47..50 "x''"
SEMICOLON@50..51 ";"
WHITESPACE@51..53 "\n\n"
COMMENT@53..62 "-- errors"
WHITESPACE@62..63 "\n"
SELECT@63..76
SELECT_CLAUSE@63..76
SELECT_KW@63..69 "select"
WHITESPACE@69..70 " "
TARGET_LIST@70..76
TARGET@70..76
LITERAL@70..76
BYTE_STRING@70..76 "x'1FZ'"
SEMICOLON@76..77 ";"
WHITESPACE@77..78 "\n"
SELECT@78..90
SELECT_CLAUSE@78..90
SELECT_KW@78..84 "select"
WHITESPACE@84..85 " "
TARGET_LIST@85..90
TARGET@85..90
LITERAL@85..90
BYTE_STRING@85..90 "x'1G'"
SEMICOLON@90..91 ";"
WHITESPACE@91..92 "\n"
SELECT@92..106
SELECT_CLAUSE@92..106
SELECT_KW@92..98 "select"
WHITESPACE@98..99 " "
TARGET_LIST@99..106
TARGET@99..106
LITERAL@99..106
BYTE_STRING@99..106 "x'1G2H'"
SEMICOLON@106..107 ";"
WHITESPACE@107..108 "\n"
SELECT@108..122
SELECT_CLAUSE@108..122
SELECT_KW@108..114 "select"
WHITESPACE@114..115 " "
TARGET_LIST@115..122
TARGET@115..122
LITERAL@115..122
BYTE_STRING@115..122 "X'GHIJ'"
SEMICOLON@122..123 ";"
WHITESPACE@123..124 "\n"

error[syntax-error]: "Z" is not a valid hexadecimal digit
╭▸
7 │ select x'1FZ';
╰╴ ━
error[syntax-error]: "G" is not a valid hexadecimal digit
╭▸
8 │ select x'1G';
╰╴ ━
error[syntax-error]: "G" is not a valid hexadecimal digit
╭▸
9 │ select x'1G2H';
╰╴ ━
error[syntax-error]: "H" is not a valid hexadecimal digit
╭▸
9 │ select x'1G2H';
╰╴ ━
error[syntax-error]: "G" is not a valid hexadecimal digit
╭▸
10 │ select X'GHIJ';
╰╴ ━
error[syntax-error]: "H" is not a valid hexadecimal digit
╭▸
10 │ select X'GHIJ';
╰╴ ━
error[syntax-error]: "I" is not a valid hexadecimal digit
╭▸
10 │ select X'GHIJ';
╰╴ ━
error[syntax-error]: "J" is not a valid hexadecimal digit
╭▸
10 │ select X'GHIJ';
╰╴ ━
Loading
Loading