feat(query): add CONV scalar function (#19892)

KKould · web-flow · commit 102392d87eac · 2026-05-21T03:35:56.000Z
feat(query): add conv scalar function
diff --git a/src/query/functions/src/scalars/binary.rs b/src/query/functions/src/scalars/binary.rs
@@ -220,17 +220,30 @@ pub fn register(registry: &mut FunctionRegistry) {
 
     registry
         .scalar_builder("to_hex")
-        .aliases(&["hex", "hex_encode"])
+        .aliases(&["hex_encode"])
         .function()
         .typed_1_arg::<BinaryType, StringType>()
         .passthrough_nullable()
         .calc_domain(|_, _| FunctionDomain::Full)
         .vectorized(vectorize_binary_to_string(
             |col| col.total_bytes_len() * 2,
             |val, output, _| {
-                let extra_len = val.len() * 2;
-                output.row_buffer.resize(extra_len, 0);
-                hex::encode_to_slice(val, &mut output.row_buffer).unwrap();
+                write_hex_lower(val, output);
+                output.commit_row();
+            },
+        ))
+        .register();
+
+    registry
+        .scalar_builder("hex")
+        .function()
+        .typed_1_arg::<BinaryType, StringType>()
+        .passthrough_nullable()
+        .calc_domain(|_, _| FunctionDomain::Full)
+        .vectorized(vectorize_binary_to_string(
+            |col| col.total_bytes_len() * 2,
+            |val, output, _| {
+                write_hex_lower(val, output);
                 output.commit_row();
             },
         ))
@@ -349,6 +362,12 @@ fn eval_unhex(val: Value<StringType>, ctx: &mut EvalContext) -> Value<BinaryType
     )(val, ctx)
 }
 
+pub fn write_hex_lower(bytes: &[u8], output: &mut StringColumnBuilder) {
+    let old_len = output.row_buffer.len();
+    output.row_buffer.resize(old_len + bytes.len() * 2, 0);
+    hex::encode_to_slice(bytes, &mut output.row_buffer[old_len..]).unwrap();
+}
+
 fn eval_from_base64(val: Value<StringType>, ctx: &mut EvalContext) -> Value<BinaryType> {
     vectorize_string_to_binary(
         |col| col.total_bytes_len() * 4 / 3 + col.len() * 4,
diff --git a/src/query/functions/src/scalars/string.rs b/src/query/functions/src/scalars/string.rs
@@ -32,6 +32,7 @@ use databend_common_expression::vectorize_with_builder_4_arg;
 use databend_functions_scalar_decimal::register_decimal_to_uuid;
 use stringslice::StringSlice;
 
+use crate::scalars::binary::write_hex_lower;
 use crate::srfs;
 
 pub const ALL_STRING_FUNC_NAMES: &[&str] = &[
@@ -60,6 +61,7 @@ pub const ALL_STRING_FUNC_NAMES: &[&str] = &[
     "trim_trailing",
     "trim_both",
     "to_hex",
+    "conv",
     "bin",
     "oct",
     "to_hex",
@@ -668,16 +670,66 @@ pub fn register(registry: &mut FunctionRegistry) {
         .calc_domain(|_, _| FunctionDomain::Full)
         .vectorized(vectorize_with_builder_1_arg::<StringType, StringType>(
             |val, output, _| {
-                let len = val.len() * 2;
-                output.row_buffer.resize(len, 0);
-                hex::encode_to_slice(val, &mut output.row_buffer).unwrap();
+                write_hex_lower(val.as_bytes(), output);
                 output.commit_row();
             },
         ))
         .register();
 
-    // TODO: generalize them to be alias of [CONV](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_conv)
-    // Tracking issue: https://github.com/datafuselabs/databend/issues/7242
+    registry
+        .scalar_builder("hex")
+        .function()
+        .typed_1_arg::<StringType, StringType>()
+        .passthrough_nullable()
+        .calc_domain(|_, _| FunctionDomain::Full)
+        .vectorized(vectorize_with_builder_1_arg::<StringType, StringType>(
+            |val, output, _| {
+                write_hex_lower(val.as_bytes(), output);
+                output.commit_row();
+            },
+        ))
+        .register();
+
+    registry.register_passthrough_nullable_3_arg::<
+        StringType,
+        NumberType<i64>,
+        NumberType<i64>,
+        StringType,
+        _,
+        _,
+    >(
+        "conv",
+        |_, _, _, _| FunctionDomain::MayThrow,
+        vectorize_with_builder_3_arg::<
+            StringType,
+            NumberType<i64>,
+            NumberType<i64>,
+            StringType,
+        >(|num, from_base, to_base, output, ctx| {
+            conv(num, from_base, to_base, output, ctx);
+        }),
+    );
+
+    registry.register_passthrough_nullable_3_arg::<
+        NumberType<i64>,
+        NumberType<i64>,
+        NumberType<i64>,
+        StringType,
+        _,
+        _,
+    >(
+        "conv",
+        |_, _, _, _| FunctionDomain::MayThrow,
+        vectorize_with_builder_3_arg::<
+            NumberType<i64>,
+            NumberType<i64>,
+            NumberType<i64>,
+            StringType,
+        >(|num, from_base, to_base, output, ctx| {
+            conv(&num.to_string(), from_base, to_base, output, ctx);
+        }),
+    );
+
     registry
         .scalar_builder("bin")
         .function()
@@ -686,7 +738,7 @@ pub fn register(registry: &mut FunctionRegistry) {
         .calc_domain(|_, _| FunctionDomain::Full)
         .vectorized(vectorize_with_builder_1_arg::<NumberType<i64>, StringType>(
             |val, output, _| {
-                write!(output.row_buffer, "{val:b}").unwrap();
+                write_conv_num(val as u64, 2, output);
                 output.commit_row();
             },
         ))
@@ -700,7 +752,7 @@ pub fn register(registry: &mut FunctionRegistry) {
         .calc_domain(|_, _| FunctionDomain::Full)
         .vectorized(vectorize_with_builder_1_arg::<NumberType<i64>, StringType>(
             |val, output, _| {
-                write!(output.row_buffer, "{val:o}").unwrap();
+                write_conv_num(val as u64, 8, output);
                 output.commit_row();
             },
         ))
@@ -720,6 +772,20 @@ pub fn register(registry: &mut FunctionRegistry) {
         ))
         .register();
 
+    registry
+        .scalar_builder("hex")
+        .function()
+        .typed_1_arg::<NumberType<i64>, StringType>()
+        .passthrough_nullable()
+        .calc_domain(|_, _| FunctionDomain::Full)
+        .vectorized(vectorize_with_builder_1_arg::<NumberType<i64>, StringType>(
+            |val, output, _| {
+                write_conv_num(val as u64, 16, output);
+                output.commit_row();
+            },
+        ))
+        .register();
+
     const MAX_REPEAT_TIMES: u64 = 1000000;
     registry.register_passthrough_nullable_2_arg::<StringType, NumberType<u64>, StringType, _, _>(
         "repeat",
@@ -1079,6 +1145,117 @@ fn substr(builder: &mut StringColumnBuilder, str: &str, pos: i64, len: u64) {
     builder.commit_row();
 }
 
+fn conv(
+    num: &str,
+    from_base: i64,
+    to_base: i64,
+    output: &mut StringColumnBuilder,
+    ctx: &mut databend_common_expression::EvalContext,
+) {
+    let Some(from_base) = validate_conv_base(from_base) else {
+        ctx.set_error(
+            output.len(),
+            format!(
+                "from_base absolute value must be between 2 and 36, but got {}",
+                from_base
+            ),
+        );
+        output.commit_row();
+        return;
+    };
+
+    let signed_to_base = to_base < 0;
+    let Some(to_base) = validate_conv_base(to_base) else {
+        ctx.set_error(
+            output.len(),
+            format!(
+                "to_base absolute value must be between 2 and 36, but got {}",
+                to_base
+            ),
+        );
+        output.commit_row();
+        return;
+    };
+
+    let mut num = parse_conv_num(num, from_base);
+    if signed_to_base {
+        let signed_num = num as i64;
+        if signed_num < 0 {
+            output.put_char('-');
+            num = (signed_num as u64).wrapping_neg();
+        }
+    }
+    write_conv_num(num, to_base, output);
+    output.commit_row();
+}
+
+fn validate_conv_base(base: i64) -> Option<u32> {
+    let base = base.unsigned_abs();
+    if (2..=36).contains(&base) {
+        Some(base as u32)
+    } else {
+        None
+    }
+}
+
+fn parse_conv_num(num: &str, from_base: u32) -> u64 {
+    let num = num.trim_start();
+    let (negative, digits) = if let Some(rest) = num.strip_prefix('-') {
+        (true, rest)
+    } else if let Some(rest) = num.strip_prefix('+') {
+        (false, rest)
+    } else {
+        (false, num)
+    };
+
+    let mut value = 0_u64;
+    for ch in digits.bytes() {
+        let Some(digit) = conv_digit(ch) else {
+            break;
+        };
+        if digit >= from_base {
+            break;
+        }
+
+        value = value
+            .saturating_mul(from_base as u64)
+            .saturating_add(digit as u64);
+    }
+
+    if negative {
+        value.wrapping_neg()
+    } else {
+        value
+    }
+}
+
+fn conv_digit(ch: u8) -> Option<u32> {
+    match ch {
+        b'0'..=b'9' => Some((ch - b'0') as u32),
+        b'a'..=b'z' => Some((ch - b'a' + 10) as u32),
+        b'A'..=b'Z' => Some((ch - b'A' + 10) as u32),
+        _ => None,
+    }
+}
+
+fn write_conv_num(mut num: u64, to_base: u32, output: &mut StringColumnBuilder) {
+    const DIGITS: &[u8; 36] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+    if num == 0 {
+        output.put_char('0');
+        return;
+    }
+
+    let mut buf = [0_u8; 64];
+    let mut idx = buf.len();
+    while num > 0 {
+        idx -= 1;
+        buf[idx] = DIGITS[(num % to_base as u64) as usize];
+        num /= to_base as u64;
+    }
+    output.put_slice(&buf[idx..]);
+}
+
 #[inline]
 fn substr_ascii(builder: &mut StringColumnBuilder, str: &str, pos: i64, len: u64) {
     let byte_len = str.len();
diff --git a/src/query/functions/tests/it/scalars/binary.rs b/src/query/functions/tests/it/scalars/binary.rs
@@ -51,6 +51,7 @@ fn test_length(file: &mut impl Write) {
 
 fn test_to_hex(file: &mut impl Write) {
     run_ast(file, "to_hex(to_binary('abc'))", &[]);
+    run_ast(file, "hex(to_binary('abc'))", &[]);
     run_ast(file, "to_hex(to_binary(a))", &[(
         "a",
         StringType::from_data(vec!["abc", "def", "databend"]),
diff --git a/src/query/functions/tests/it/scalars/string.rs b/src/query/functions/tests/it/scalars/string.rs
@@ -49,6 +49,7 @@ fn test_string() {
     test_bin(file);
     test_oct(file);
     test_hex(file);
+    test_conv(file);
     test_pad(file);
     test_replace(file);
     test_translate(file);
@@ -498,6 +499,34 @@ fn test_hex(file: &mut impl Write) {
     run_ast(file, "hex(c)", columns);
     run_ast(file, "hex(d)", columns);
     run_ast(file, "hex(e)", columns);
+    run_ast(file, "to_hex(c)", columns);
+    run_ast(file, "to_hex(e)", columns);
+}
+
+fn test_conv(file: &mut impl Write) {
+    run_ast(file, "conv('a', 16, 10)", &[]);
+    run_ast(file, "conv('6E', 18, 8)", &[]);
+    run_ast(file, "conv('zz', 36, 10)", &[]);
+    run_ast(file, "conv('11', 2, 36)", &[]);
+    run_ast(file, "conv(10, 10, 2)", &[]);
+    run_ast(file, "conv(10, 16, 10)", &[]);
+    run_ast(file, "conv(15, 10, 16)", &[]);
+    run_ast(file, "conv(-128, 10, 8)", &[]);
+    run_ast(file, "conv(-17, 10, -18)", &[]);
+    run_ast(file, "conv('FFFFFFFFFFFFFFFF', 16, -10)", &[]);
+    run_ast(file, "conv('11', -2, 10)", &[]);
+
+    let columns = &[
+        ("a", StringType::from_data(vec!["10", "a", "zz"])),
+        (
+            "b",
+            Int64Type::from_data_with_validity(vec![2i64, 16, 36], vec![true, false, true]),
+        ),
+        ("c", Int64Type::from_data(vec![10i64, 10, 16])),
+        ("d", Int64Type::from_data(vec![10i64, 16, -128])),
+    ];
+    run_ast(file, "conv(a, b, c)", columns);
+    run_ast(file, "conv(d, 10, 2)", columns);
 }
 
 fn test_pad(file: &mut impl Write) {
diff --git a/src/query/functions/tests/it/scalars/testdata/binary.txt b/src/query/functions/tests/it/scalars/testdata/binary.txt
@@ -95,6 +95,15 @@ output domain  : {"616263"..="616263"}
 output         : '616263'
 
 
+ast            : hex(to_binary('abc'))
+raw expr       : hex(to_binary('abc'))
+checked expr   : hex<Binary>(CAST<String>("abc" AS Binary))
+optimized expr : "616263"
+output type    : String
+output domain  : {"616263"..="616263"}
+output         : '616263'
+
+
 ast            : to_hex(to_binary(a))
 raw expr       : to_hex(to_binary(a::String))
 checked expr   : to_hex<Binary>(CAST<String>(a AS Binary))
diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt
@@ -31,7 +31,6 @@ diff_dows -> diff_days
 diff_doys -> diff_days
 diff_epochs -> diff_seconds
 diff_isodows -> diff_days
-hex -> to_hex
 hex_decode_binary -> from_hex
 hex_encode -> to_hex
 intdiv -> div
@@ -1355,6 +1354,10 @@ Functions overloads:
 28 contains(Array(Boolean), Boolean) :: Boolean
 29 contains(Array(Boolean) NULL, Boolean NULL) :: Boolean NULL
 30 contains(Array(T0) NULL, T0) :: Boolean
+0 conv(String, Int64, Int64) :: String
+1 conv(String NULL, Int64 NULL, Int64 NULL) :: String NULL
+2 conv(Int64, Int64, Int64) :: String
+3 conv(Int64 NULL, Int64 NULL, Int64 NULL) :: String NULL
 0 convert_timezone(String, Timestamp) :: Timestamp
 1 convert_timezone(String NULL, Timestamp NULL) :: Timestamp NULL
 0 cos(Float64) :: Float64
@@ -2259,6 +2262,12 @@ Functions overloads:
 1 h3_unidirectional_edge_is_valid(UInt64 NULL) :: Boolean NULL
 0 haversine(Float64, Float64, Float64, Float64) :: Float64
 1 haversine(Float64 NULL, Float64 NULL, Float64 NULL, Float64 NULL) :: Float64 NULL
+0 hex(String) :: String
+1 hex(String NULL) :: String NULL
+2 hex(Int64) :: String
+3 hex(Int64 NULL) :: String NULL
+4 hex(Binary) :: String
+5 hex(Binary NULL) :: String NULL
 0 hilbert_range_index FACTORY
 0 humanize_number(Float64) :: String
 1 humanize_number(Float64 NULL) :: String NULL
diff --git a/src/query/functions/tests/it/scalars/testdata/string.txt b/src/query/functions/tests/it/scalars/testdata/string.txt
diff --git a/tests/sqllogictests/suites/query/functions/02_0019_function_strings_hex.test b/tests/sqllogictests/suites/query/functions/02_0019_function_strings_hex.test
diff --git a/tests/sqllogictests/suites/query/functions/02_0084_function_strings_conv.test b/tests/sqllogictests/suites/query/functions/02_0084_function_strings_conv.test
