Skip to content

Commit 4d1552a

Browse files
aaarone90aaarone90
committed
Fixing integration tests
Signed-off-by: Aaron Alvarez <aaarone@amazon.com>
1 parent ea02f53 commit 4d1552a

20 files changed

Lines changed: 642 additions & 186 deletions

File tree

‎core/src/main/java/org/opensearch/sql/expression/function/udf/AutoConvertFunction.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,11 @@ public AutoConvertFunction() {
3030

3131
@Override
3232
public SqlReturnTypeInference getReturnTypeInference() {
33-
return ReturnTypes.DOUBLE_FORCE_NULLABLE;
33+
return ReturnTypes.explicit(
34+
factory ->
35+
factory.createTypeWithNullability(
36+
factory.createSqlType(org.apache.calcite.sql.type.SqlTypeName.DECIMAL, 38, 10),
37+
true));
3438
}
3539

3640
@Override
@@ -43,7 +47,8 @@ public static class AutoConvertImplementor implements NotNullImplementor {
4347
public Expression implement(
4448
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
4549
Expression fieldValue = translatedOperands.get(0);
46-
return Expressions.call(ConversionUtils.class, "autoConvert", fieldValue);
50+
Expression result = Expressions.call(ConversionUtils.class, "autoConvert", fieldValue);
51+
return Expressions.convert_(result, Number.class);
4752
}
4853
}
4954
}

‎core/src/main/java/org/opensearch/sql/expression/function/udf/ConversionUtils.java‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import java.util.regex.Pattern;
9+
10+
/** Utility class for conversion functions used by convert command UDFs. */
11+
public class ConversionUtils {
12+
13+
private static final Pattern COMMA_PATTERN = Pattern.compile(",");
14+
private static final Pattern LEADING_NUMBER_PATTERN = Pattern.compile("^(\\d+(?:\\.\\d+)?)");
15+
16+
/**
17+
* Auto convert field value to numeric type using best-fit heuristics. Tries conversions in order:
18+
* direct numeric, remove commas, extract leading numbers. Returns Long for integers, Double for
19+
* decimals.
20+
*/
21+
public static Object autoConvert(Object value) {
22+
if (value == null) return null;
23+
24+
// If already a number, return as-is (preserve Long for integers, Double for decimals)
25+
if (value instanceof Long
26+
|| value instanceof Integer
27+
|| value instanceof Short
28+
|| value instanceof Byte) {
29+
return ((Number) value).longValue();
30+
}
31+
if (value instanceof Double || value instanceof Float) {
32+
return ((Number) value).doubleValue();
33+
}
34+
35+
String str = value.toString().trim();
36+
if (str.isEmpty()) return null;
37+
38+
// Step 1: Try direct number conversion first (num() functionality)
39+
try {
40+
if (str.contains(".")) {
41+
return Double.parseDouble(str);
42+
} else {
43+
return Long.parseLong(str);
44+
}
45+
} catch (NumberFormatException e) {
46+
// Step 2: Try removing commas then convert (rmcomma() + num())
47+
String noCommas = COMMA_PATTERN.matcher(str).replaceAll("");
48+
try {
49+
if (noCommas.contains(".")) {
50+
return Double.parseDouble(noCommas);
51+
} else {
52+
return Long.parseLong(noCommas);
53+
}
54+
} catch (NumberFormatException e2) {
55+
// Step 3: Try extracting leading numbers (rmunit() functionality)
56+
var matcher = LEADING_NUMBER_PATTERN.matcher(noCommas);
57+
if (matcher.find()) {
58+
String numberStr = matcher.group(1);
59+
try {
60+
if (numberStr.contains(".")) {
61+
return Double.parseDouble(numberStr);
62+
} else {
63+
return Long.parseLong(numberStr);
64+
}
65+
} catch (NumberFormatException e3) {
66+
return null;
67+
}
68+
}
69+
return null;
70+
}
71+
}
72+
}
73+
74+
/** Convert field value to number. Returns Long for integers, Double for decimals. */
75+
public static Object numConvert(Object value) {
76+
if (value == null) return null;
77+
78+
// If already a number, return as-is (preserve Long for integers, Double for decimals)
79+
if (value instanceof Long
80+
|| value instanceof Integer
81+
|| value instanceof Short
82+
|| value instanceof Byte) {
83+
return ((Number) value).longValue();
84+
}
85+
if (value instanceof Double || value instanceof Float) {
86+
return ((Number) value).doubleValue();
87+
}
88+
89+
String str = value.toString().trim();
90+
if (str.isEmpty()) return null;
91+
92+
try {
93+
if (str.contains(".")) {
94+
return Double.parseDouble(str);
95+
} else {
96+
return Long.parseLong(str);
97+
}
98+
} catch (NumberFormatException e) {
99+
return null;
100+
}
101+
}
102+
103+
/** Remove commas from field value. */
104+
public static Object rmcommaConvert(Object value) {
105+
if (value == null) return null;
106+
return COMMA_PATTERN.matcher(value.toString()).replaceAll("");
107+
}
108+
109+
/** Extract leading numbers and remove trailing text. */
110+
public static Object rmunitConvert(Object value) {
111+
if (value == null) return null;
112+
String str = value.toString().trim();
113+
if (str.isEmpty()) return null;
114+
115+
var matcher = LEADING_NUMBER_PATTERN.matcher(str);
116+
if (matcher.find()) {
117+
String numberStr = matcher.group(1);
118+
try {
119+
if (numberStr.contains(".")) {
120+
return Double.parseDouble(numberStr);
121+
} else {
122+
return Long.parseLong(numberStr);
123+
}
124+
} catch (NumberFormatException e) {
125+
return null;
126+
}
127+
}
128+
return null;
129+
}
130+
131+
// Overloaded methods for specific types
132+
public static Object autoConvert(String value) {
133+
return autoConvert((Object) value);
134+
}
135+
136+
public static Object autoConvert(long value) {
137+
return autoConvert((Object) value);
138+
}
139+
140+
public static Object autoConvert(Long value) {
141+
return autoConvert((Object) value);
142+
}
143+
144+
public static Object autoConvert(double value) {
145+
return autoConvert((Object) value);
146+
}
147+
148+
public static Object autoConvert(Double value) {
149+
return autoConvert((Object) value);
150+
}
151+
152+
public static Object numConvert(String value) {
153+
return numConvert((Object) value);
154+
}
155+
156+
public static Object numConvert(long value) {
157+
return numConvert((Object) value);
158+
}
159+
160+
public static Object numConvert(Long value) {
161+
return numConvert((Object) value);
162+
}
163+
164+
public static Object numConvert(double value) {
165+
return numConvert((Object) value);
166+
}
167+
168+
public static Object numConvert(Double value) {
169+
return numConvert((Object) value);
170+
}
171+
172+
public static Object rmcommaConvert(String value) {
173+
return rmcommaConvert((Object) value);
174+
}
175+
176+
public static Object rmunitConvert(String value) {
177+
return rmunitConvert((Object) value);
178+
}
179+
}

‎core/src/main/java/org/opensearch/sql/expression/function/udf/NumConvertFunction.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,11 @@ public NumConvertFunction() {
3030

3131
@Override
3232
public SqlReturnTypeInference getReturnTypeInference() {
33-
return ReturnTypes.DOUBLE_FORCE_NULLABLE;
33+
return ReturnTypes.explicit(
34+
factory ->
35+
factory.createTypeWithNullability(
36+
factory.createSqlType(org.apache.calcite.sql.type.SqlTypeName.DECIMAL, 38, 10),
37+
true));
3438
}
3539

3640
@Override
@@ -43,7 +47,8 @@ public static class NumConvertImplementor implements NotNullImplementor {
4347
public Expression implement(
4448
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
4549
Expression fieldValue = translatedOperands.get(0);
46-
return Expressions.call(ConversionUtils.class, "numConvert", fieldValue);
50+
Expression result = Expressions.call(ConversionUtils.class, "numConvert", fieldValue);
51+
return Expressions.convert_(result, Number.class);
4752
}
4853
}
4954
}

‎core/src/main/java/org/opensearch/sql/expression/function/udf/RmcommaConvertFunction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public RmcommaConvertFunction() {
2929

3030
@Override
3131
public SqlReturnTypeInference getReturnTypeInference() {
32-
return ReturnTypes.DOUBLE_FORCE_NULLABLE;
32+
return ReturnTypes.VARCHAR_FORCE_NULLABLE;
3333
}
3434

3535
@Override

‎core/src/main/java/org/opensearch/sql/expression/function/udf/RmunitConvertFunction.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ public RmunitConvertFunction() {
3030

3131
@Override
3232
public SqlReturnTypeInference getReturnTypeInference() {
33-
return ReturnTypes.DOUBLE_FORCE_NULLABLE;
33+
return ReturnTypes.explicit(
34+
factory ->
35+
factory.createTypeWithNullability(
36+
factory.createSqlType(org.apache.calcite.sql.type.SqlTypeName.BIGINT), true));
3437
}
3538

3639
@Override
@@ -43,7 +46,8 @@ public static class RmunitConvertImplementor implements NotNullImplementor {
4346
public Expression implement(
4447
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
4548
Expression fieldValue = translatedOperands.get(0);
46-
return Expressions.call(ConversionUtils.class, "rmunitConvert", fieldValue);
49+
Expression result = Expressions.call(ConversionUtils.class, "rmunitConvert", fieldValue);
50+
return Expressions.convert_(result, Number.class);
4751
}
4852
}
4953
}

‎core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionUtilsTest.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import static org.junit.jupiter.api.Assertions.assertEquals;
9+
import static org.junit.jupiter.api.Assertions.assertNull;
10+
11+
import org.junit.jupiter.api.Test;
12+
13+
/** Unit tests for ConversionUtils. */
14+
public class ConversionUtilsTest {
15+
16+
@Test
17+
public void testAutoConvertBasicNumbers() {
18+
// Should convert directly without any preprocessing
19+
assertEquals(123L, ConversionUtils.autoConvert("123"));
20+
assertEquals(123.45, ConversionUtils.autoConvert("123.45"));
21+
assertEquals(0L, ConversionUtils.autoConvert("0"));
22+
assertEquals(-123L, ConversionUtils.autoConvert("-123"));
23+
}
24+
25+
@Test
26+
public void testAutoConvertOptimalPath() {
27+
// Verify that simple numbers take the fastest path (no comma processing)
28+
assertEquals(42L, ConversionUtils.autoConvert("42"));
29+
assertEquals(3.14, ConversionUtils.autoConvert("3.14"));
30+
}
31+
32+
@Test
33+
public void testAutoConvertWithCommas() {
34+
// Should fail direct conversion, then succeed with comma removal
35+
assertEquals(1234L, ConversionUtils.autoConvert("1,234"));
36+
assertEquals(1234.56, ConversionUtils.autoConvert("1,234.56"));
37+
assertEquals(1000000L, ConversionUtils.autoConvert("1,000,000"));
38+
}
39+
40+
@Test
41+
public void testAutoConvertWithUnits() {
42+
// Should fail direct and comma removal, then succeed with unit extraction
43+
assertEquals(123L, ConversionUtils.autoConvert("123 dollars"));
44+
assertEquals(45.67, ConversionUtils.autoConvert("45.67 kg"));
45+
assertEquals(100L, ConversionUtils.autoConvert("100ms"));
46+
}
47+
48+
@Test
49+
public void testAutoConvertCombined() {
50+
// Should fail direct and comma removal, then succeed with unit extraction
51+
assertEquals(1234L, ConversionUtils.autoConvert("1,234 dollars"));
52+
assertEquals(5678.90, ConversionUtils.autoConvert("5,678.90 USD"));
53+
}
54+
55+
@Test
56+
public void testAutoConvertNullAndEmpty() {
57+
assertNull(ConversionUtils.autoConvert((Object) null));
58+
assertNull(ConversionUtils.autoConvert(""));
59+
assertNull(ConversionUtils.autoConvert(" "));
60+
}
61+
62+
@Test
63+
public void testAutoConvertInvalid() {
64+
assertNull(ConversionUtils.autoConvert("abc"));
65+
assertNull(ConversionUtils.autoConvert("no numbers here"));
66+
}
67+
68+
@Test
69+
public void testNumConvert() {
70+
assertEquals(123L, ConversionUtils.numConvert("123"));
71+
assertEquals(123.45, ConversionUtils.numConvert("123.45"));
72+
assertNull(ConversionUtils.numConvert("1,234")); // Should fail with commas
73+
assertNull(ConversionUtils.numConvert("123 dollars")); // Should fail with text
74+
}
75+
76+
@Test
77+
public void testRmcommaConvert() {
78+
assertEquals("1234", ConversionUtils.rmcommaConvert("1,234"));
79+
assertEquals("1234.56", ConversionUtils.rmcommaConvert("1,234.56"));
80+
assertEquals("abc", ConversionUtils.rmcommaConvert("abc"));
81+
}
82+
83+
@Test
84+
public void testRmunitConvert() {
85+
assertEquals(123L, ConversionUtils.rmunitConvert("123 dollars"));
86+
assertEquals(45.67, ConversionUtils.rmunitConvert("45.67 kg"));
87+
assertNull(ConversionUtils.rmunitConvert("no numbers"));
88+
}
89+
}

0 commit comments

Comments
 (0)