diff --git a/src/query/functions/src/scalars/binary.rs b/src/query/functions/src/scalars/binary.rs index 9c7e54a84239c..fe20ca995f540 100644 --- a/src/query/functions/src/scalars/binary.rs +++ b/src/query/functions/src/scalars/binary.rs @@ -172,12 +172,16 @@ pub fn register(registry: &mut FunctionRegistry) { match format.to_ascii_lowercase().as_str() { "hex" => eval_unhex(val, ctx), "base64" => eval_from_base64(val, ctx), + "latin1" | "latin-1" | "iso-8859-1" => eval_latin1_bytes(val, ctx), "utf-8" => match val { Value::Scalar(val) => Value::Scalar(val.as_bytes().to_vec()), Value::Column(col) => Value::Column(col.into()), }, _ => { - ctx.set_error(0, "The format option only supports hex, base64, and utf-8"); + ctx.set_error( + 0, + "The format option only supports hex, base64, latin1, and utf-8", + ); Value::Scalar(Vec::new()) } } @@ -206,6 +210,7 @@ pub fn register(registry: &mut FunctionRegistry) { match format.to_ascii_lowercase().as_str() { "hex" => error_to_null(eval_unhex)(val, ctx), "base64" => error_to_null(eval_from_base64)(val, ctx), + "latin1" | "latin-1" | "iso-8859-1" => error_to_null(eval_latin1_bytes)(val, ctx), "utf-8" => match val { Value::Scalar(val) => Value::Scalar(Some(val.as_bytes().to_vec())), Value::Column(col) => { @@ -347,6 +352,26 @@ fn eval_utf8_bytes_nullable(val: Value) -> Value, ctx: &mut EvalContext) -> Value { + vectorize_string_to_binary( + |col| col.total_bytes_len(), + |val, output, ctx| { + for ch in val.chars() { + let code_point = u32::from(ch); + if code_point > u32::from(u8::MAX) { + ctx.set_error( + output.len(), + format!("character U+{code_point:04X} cannot be encoded as latin1"), + ); + break; + } + output.data.push(code_point as u8); + } + output.commit_row(); + }, + )(val, ctx) +} + fn eval_unhex(val: Value, ctx: &mut EvalContext) -> Value { vectorize_string_to_binary( |col| col.total_bytes_len() / 2, diff --git a/src/query/functions/tests/it/scalars/binary.rs b/src/query/functions/tests/it/scalars/binary.rs index ad0eb023b98f2..0b63b033869f5 100644 --- a/src/query/functions/tests/it/scalars/binary.rs +++ b/src/query/functions/tests/it/scalars/binary.rs @@ -110,6 +110,29 @@ fn test_to_jsonb_binary(file: &mut impl Write) { fn test_to_binary(file: &mut impl Write, is_try: bool) { let prefix = if is_try { "TRY_" } else { "" }; + run_ast( + file, + format!("to_hex({prefix}to_binary(char(0, 65, 127, 128, 255), 'latin1'))"), + &[], + ); + run_ast( + file, + format!("to_hex({prefix}to_binary(char(128, 255), 'latin-1'))"), + &[], + ); + run_ast( + file, + format!("to_hex({prefix}to_binary(char(128, 255), 'iso-8859-1'))"), + &[], + ); + run_ast(file, format!("{prefix}to_binary(char(256), 'latin1')"), &[]); + run_ast(file, format!("to_hex({prefix}to_binary(s, 'latin1'))"), &[ + ( + "s", + StringType::from_data(vec!["\u{80}\u{ff}", "\u{100}", "plain"]), + ), + ]); + run_ast(file, format!("{prefix}to_binary(to_bitmap('1,2,3'))"), &[]); run_ast( file, diff --git a/src/query/functions/tests/it/scalars/testdata/binary.txt b/src/query/functions/tests/it/scalars/testdata/binary.txt index a9504e7a2d5d2..8c0f041f97bed 100644 --- a/src/query/functions/tests/it/scalars/testdata/binary.txt +++ b/src/query/functions/tests/it/scalars/testdata/binary.txt @@ -248,6 +248,49 @@ evaluation (internal): +--------+--------------------------------------------------------+ +ast : to_hex(to_binary(char(0, 65, 127, 128, 255), 'latin1')) +raw expr : to_hex(to_binary(char(0, 65, 127, 128, 255), 'latin1')) +checked expr : to_hex(to_binary(char(CAST(0_u8 AS Int64), CAST(65_u8 AS Int64), CAST(127_u8 AS Int64), CAST(128_u8 AS Int64), CAST(255_u8 AS Int64)), "latin1")) +optimized expr : "00417f80ff" +output type : String +output domain : {"00417f80ff"..="00417f80ff"} +output : '00417f80ff' + + +ast : to_hex(to_binary(char(128, 255), 'latin-1')) +raw expr : to_hex(to_binary(char(128, 255), 'latin-1')) +checked expr : to_hex(to_binary(char(CAST(128_u8 AS Int64), CAST(255_u8 AS Int64)), "latin-1")) +optimized expr : "80ff" +output type : String +output domain : {"80ff"..="80ff"} +output : '80ff' + + +ast : to_hex(to_binary(char(128, 255), 'iso-8859-1')) +raw expr : to_hex(to_binary(char(128, 255), 'iso-8859-1')) +checked expr : to_hex(to_binary(char(CAST(128_u8 AS Int64), CAST(255_u8 AS Int64)), "iso-8859-1")) +optimized expr : "80ff" +output type : String +output domain : {"80ff"..="80ff"} +output : '80ff' + + +error: + --> SQL:1:1 + | +1 | to_binary(char(256), 'latin1') + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ character U+0100 cannot be encoded as latin1 while evaluating function `to_binary('Ā', 'latin1')` in expr `to_binary(char(CAST(256 AS Int64)), 'latin1')` + + + +error: + --> SQL:1:8 + | +1 | to_hex(to_binary(s, 'latin1')) + | ^^^^^^^^^^^^^^^^^^^^^^ character U+0100 cannot be encoded as latin1 while evaluating function `to_binary('Ā', 'latin1')` in expr `to_binary(s, 'latin1')`, during run expr: `to_hex(to_binary(s, 'latin1'))` + + + ast : to_binary(to_bitmap('1,2,3')) raw expr : to_binary(to_bitmap('1,2,3')) checked expr : CAST(CAST("1,2,3" AS Bitmap) AS Binary) @@ -381,6 +424,64 @@ evaluation (internal): +--------+--------------------------------------------------------+ +ast : to_hex(TRY_to_binary(char(0, 65, 127, 128, 255), 'latin1')) +raw expr : to_hex(TRY_to_binary(char(0, 65, 127, 128, 255), 'latin1')) +checked expr : to_hex(try_to_binary(char(CAST(0_u8 AS Int64), CAST(65_u8 AS Int64), CAST(127_u8 AS Int64), CAST(128_u8 AS Int64), CAST(255_u8 AS Int64)), "latin1")) +optimized expr : "00417f80ff" +output type : String NULL +output domain : {"00417f80ff"..="00417f80ff"} +output : '00417f80ff' + + +ast : to_hex(TRY_to_binary(char(128, 255), 'latin-1')) +raw expr : to_hex(TRY_to_binary(char(128, 255), 'latin-1')) +checked expr : to_hex(try_to_binary(char(CAST(128_u8 AS Int64), CAST(255_u8 AS Int64)), "latin-1")) +optimized expr : "80ff" +output type : String NULL +output domain : {"80ff"..="80ff"} +output : '80ff' + + +ast : to_hex(TRY_to_binary(char(128, 255), 'iso-8859-1')) +raw expr : to_hex(TRY_to_binary(char(128, 255), 'iso-8859-1')) +checked expr : to_hex(try_to_binary(char(CAST(128_u8 AS Int64), CAST(255_u8 AS Int64)), "iso-8859-1")) +optimized expr : "80ff" +output type : String NULL +output domain : {"80ff"..="80ff"} +output : '80ff' + + +ast : TRY_to_binary(char(256), 'latin1') +raw expr : TRY_to_binary(char(256), 'latin1') +checked expr : try_to_binary(char(CAST(256_u16 AS Int64)), "latin1") +optimized expr : NULL +output type : Binary NULL +output domain : {NULL} +output : NULL + + +ast : to_hex(TRY_to_binary(s, 'latin1')) +raw expr : to_hex(TRY_to_binary(s::String, 'latin1')) +checked expr : to_hex(try_to_binary(s, "latin1")) +evaluation: ++--------+-----------------+-----------------+ +| | s | Output | ++--------+-----------------+-----------------+ +| Type | String | String NULL | +| Domain | {"plain"..="Ā"} | {""..} ∪ {NULL} | +| Row 0 | '€ÿ' | '80ff' | +| Row 1 | 'Ā' | NULL | +| Row 2 | 'plain' | '706c61696e' | ++--------+-----------------+-----------------+ +evaluation (internal): ++--------+-------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-------------------------------------------------------------------------------------+ +| s | Column(StringColumn[€ÿ, Ā, plain]) | +| Output | NullableColumn { column: StringColumn[80ff, , 706c61696e], validity: [0b_____101] } | ++--------+-------------------------------------------------------------------------------------+ + + ast : TRY_to_binary(to_bitmap('1,2,3')) raw expr : TRY_to_binary(to_bitmap('1,2,3')) checked expr : TRY_CAST(CAST("1,2,3" AS Bitmap) AS Binary NULL) diff --git a/tests/sqllogictests/suites/base/03_common/03_0041_insert_into_binary.test b/tests/sqllogictests/suites/base/03_common/03_0041_insert_into_binary.test index 5b2f4cc015649..8f065980b4f9a 100644 --- a/tests/sqllogictests/suites/base/03_common/03_0041_insert_into_binary.test +++ b/tests/sqllogictests/suites/base/03_common/03_0041_insert_into_binary.test @@ -27,6 +27,31 @@ SELECT id, v FROM t1 order by id 6 616161 7 616161 +query T +SELECT to_hex(to_binary(char(0, 65, 127, 128, 255), 'latin1')) +---- +00417f80ff + +query T +SELECT to_hex(to_binary(as_string(get(parse_json('{"v":"\u0000A\u007f\u0080\u00ff"}'), 'v')), 'latin1')) +---- +00417f80ff + +query T +SELECT to_hex(try_to_binary(char(128, 255), 'latin-1')) +---- +80ff + +query T +SELECT to_hex(try_to_binary(char(128, 255), 'iso-8859-1')) +---- +80ff + +query ? +SELECT try_to_binary(char(256), 'latin1') +---- +NULL + statement ok ALTER TABLE t1 MODIFY COLUMN v string