Skip to content

Commit ea02f53

Browse files
aaarone90aaarone90
committed
feat: Implement PPL convert command with 5 conversion functions
- Added convert command syntax and AST nodes (Convert, ConvertFunction) - Implemented 5 conversion functions: auto, num, rmcomma, rmunit, none - Full Calcite pushdown support via CalciteRelNodeVisitor - Logical and physical operators (LogicalConvert, ConvertOperator) - Comprehensive test coverage (26 tests total): * 13 unit tests (CalcitePPLConvertTest) * 8 integration tests with pushdown (ConvertCommandIT) * 8 non-pushdown tests (CalciteConvertCommandIT) * 3 explain tests (ExplainIT) * 2 cross-cluster tests (CrossClusterSearchIT) * Anonymizer and v2 unsupported tests - User documentation (docs/user/ppl/cmd/convert.md) - Code cleanup: removed dead code, simplified javadocs - Version: 3.5 (experimental) All tests compile and pass successfully. Signed-off-by: Aaron Alvarez <aaarone@amazon.com>
1 parent 74b2fb3 commit ea02f53

32 files changed

Lines changed: 1306 additions & 2 deletions

‎core/src/main/java/org/opensearch/sql/analysis/Analyzer.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,6 +523,48 @@ public LogicalPlan visitEval(Eval node, AnalysisContext context) {
523523
return new LogicalEval(child, expressionsBuilder.build());
524524
}
525525

526+
/** Build {@link org.opensearch.sql.planner.logical.LogicalConvert}. */
527+
@Override
528+
public LogicalPlan visitConvert(
529+
org.opensearch.sql.ast.tree.Convert node, AnalysisContext context) {
530+
LogicalPlan child = node.getChild().get(0).accept(this, context);
531+
ImmutableList.Builder<Pair<ReferenceExpression, Expression>> conversionsBuilder =
532+
new Builder<>();
533+
534+
for (org.opensearch.sql.ast.tree.ConvertFunction convertFunc : node.getConvertFunctions()) {
535+
String functionName = convertFunc.getFunctionName();
536+
List<String> fieldList = convertFunc.getFieldList();
537+
String asField = convertFunc.getAsField();
538+
539+
// Process each field in the conversion function
540+
for (String fieldName : fieldList) {
541+
// Analyze the field reference
542+
Expression fieldExpr = expressionAnalyzer.analyze(AstDSL.field(fieldName), context);
543+
544+
// Build the conversion function call
545+
// For now, we'll create a simple function call - this will be expanded later
546+
// to properly map conversion function names to actual implementations
547+
Expression conversionExpr =
548+
expressionAnalyzer.analyze(
549+
new Function(functionName, Collections.singletonList(AstDSL.field(fieldName))),
550+
context);
551+
552+
// Determine the target field name
553+
String targetFieldName = (asField != null) ? asField : fieldName;
554+
ReferenceExpression ref = DSL.ref(targetFieldName, conversionExpr.type());
555+
556+
conversionsBuilder.add(ImmutablePair.of(ref, conversionExpr));
557+
558+
// Define the new reference in type environment
559+
TypeEnvironment typeEnvironment = context.peek();
560+
typeEnvironment.define(ref);
561+
}
562+
}
563+
564+
return new org.opensearch.sql.planner.logical.LogicalConvert(
565+
child, conversionsBuilder.build(), node.getTimeformat());
566+
}
567+
526568
@Override
527569
public LogicalPlan visitAddTotals(AddTotals node, AnalysisContext context) {
528570
throw getOnlyForCalciteException("addtotals");

‎core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.opensearch.sql.ast.tree.Bin;
5555
import org.opensearch.sql.ast.tree.Chart;
5656
import org.opensearch.sql.ast.tree.CloseCursor;
57+
import org.opensearch.sql.ast.tree.Convert;
5758
import org.opensearch.sql.ast.tree.Dedupe;
5859
import org.opensearch.sql.ast.tree.Eval;
5960
import org.opensearch.sql.ast.tree.Expand;
@@ -410,6 +411,10 @@ public T visitFillNull(FillNull fillNull, C context) {
410411
return visitChildren(fillNull, context);
411412
}
412413

414+
public T visitConvert(Convert node, C context) {
415+
return visitChildren(node, context);
416+
}
417+
413418
public T visitPatterns(Patterns patterns, C context) {
414419
return visitChildren(patterns, context);
415420
}

‎core/src/main/java/org/opensearch/sql/ast/tree/Convert.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.ast.tree;
7+
8+
import com.google.common.collect.ImmutableList;
9+
import java.util.List;
10+
import lombok.EqualsAndHashCode;
11+
import lombok.Getter;
12+
import lombok.RequiredArgsConstructor;
13+
import lombok.Setter;
14+
import lombok.ToString;
15+
import org.opensearch.sql.ast.AbstractNodeVisitor;
16+
17+
/**
18+
* AST node representing the Convert command.
19+
*
20+
* <p>Syntax: convert [timeformat="format"] function(fields) [AS alias], ...
21+
*
22+
* <p>Example: convert auto(age), num(price) AS numeric_price
23+
*/
24+
@Getter
25+
@Setter
26+
@ToString
27+
@EqualsAndHashCode(callSuper = false)
28+
@RequiredArgsConstructor
29+
public class Convert extends UnresolvedPlan {
30+
/** Reserved for future time conversion functions (ctime, mktime, mstime). */
31+
private final String timeformat;
32+
33+
/** List of conversion functions to apply. */
34+
private final List<ConvertFunction> convertFunctions;
35+
36+
/** Child plan node. */
37+
private UnresolvedPlan child;
38+
39+
@Override
40+
public Convert attach(UnresolvedPlan child) {
41+
this.child = child;
42+
return this;
43+
}
44+
45+
@Override
46+
public List<UnresolvedPlan> getChild() {
47+
return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
48+
}
49+
50+
@Override
51+
public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
52+
return nodeVisitor.visitConvert(this, context);
53+
}
54+
}

‎core/src/main/java/org/opensearch/sql/ast/tree/ConvertFunction.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.ast.tree;
7+
8+
import java.util.List;
9+
import lombok.EqualsAndHashCode;
10+
import lombok.Getter;
11+
import lombok.RequiredArgsConstructor;
12+
import lombok.ToString;
13+
14+
/**
15+
* Represents a single conversion function within a convert command.
16+
*
17+
* <p>Example: auto(field1, field2) AS converted_field
18+
*/
19+
@Getter
20+
@ToString
21+
@EqualsAndHashCode
22+
@RequiredArgsConstructor
23+
public class ConvertFunction {
24+
/** The name of the conversion function (e.g., "auto", "num", "ctime"). */
25+
private final String functionName;
26+
27+
/** The list of field names or patterns to convert. */
28+
private final List<String> fieldList;
29+
30+
/** Optional alias for the converted field (AS clause). Null if not specified. */
31+
private final String asField;
32+
}

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -890,6 +890,63 @@ public RelNode visitEval(Eval node, CalcitePlanContext context) {
890890
return context.relBuilder.peek();
891891
}
892892

893+
@Override
894+
public RelNode visitConvert(
895+
org.opensearch.sql.ast.tree.Convert node, CalcitePlanContext context) {
896+
visitChildren(node, context);
897+
898+
// Build maps to track conversions
899+
java.util.Map<String, RexNode> replacements =
900+
new java.util.HashMap<>(); // field -> converted (no alias)
901+
List<Pair<String, RexNode>> additions = new ArrayList<>(); // new fields to add (with alias)
902+
903+
for (org.opensearch.sql.ast.tree.ConvertFunction convertFunc : node.getConvertFunctions()) {
904+
String functionName = convertFunc.getFunctionName();
905+
List<String> fieldList = convertFunc.getFieldList();
906+
String asField = convertFunc.getAsField();
907+
908+
// Process each field in the field list
909+
for (String fieldName : fieldList) {
910+
RexNode field = context.relBuilder.field(fieldName);
911+
912+
// Create the conversion function call
913+
RexNode convertCall =
914+
PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, functionName, field);
915+
916+
if (asField != null) {
917+
// With alias: add as new field at the end
918+
additions.add(Pair.of(asField, context.relBuilder.alias(convertCall, asField)));
919+
} else {
920+
// Without alias: replace original field in-place
921+
replacements.put(fieldName, context.relBuilder.alias(convertCall, fieldName));
922+
}
923+
}
924+
}
925+
926+
// Build projection maintaining original field order, then add new fields
927+
List<String> originalFields = context.relBuilder.peek().getRowType().getFieldNames();
928+
List<RexNode> projectList = new ArrayList<>();
929+
930+
// First, project all original fields (with replacements where applicable)
931+
for (String fieldName : originalFields) {
932+
if (replacements.containsKey(fieldName)) {
933+
// Use the converted expression for this field
934+
projectList.add(replacements.get(fieldName));
935+
} else {
936+
// Keep the original field
937+
projectList.add(context.relBuilder.field(fieldName));
938+
}
939+
}
940+
941+
// Then add new aliased fields at the end
942+
for (Pair<String, RexNode> addition : additions) {
943+
projectList.add(addition.getRight());
944+
}
945+
946+
context.relBuilder.project(projectList);
947+
return context.relBuilder.peek();
948+
}
949+
893950
private void projectPlusOverriding(
894951
List<RexNode> newFields, List<String> newNames, CalcitePlanContext context) {
895952
List<String> originalFieldNames = context.relBuilder.peek().getRowType().getFieldNames();

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,18 @@ public enum BuiltinFunctionName {
299299

300300
INTERVAL(FunctionName.of("interval")),
301301

302+
/** PPL Convert Command Functions. */
303+
AUTO(FunctionName.of("auto")),
304+
NUM(FunctionName.of("num")),
305+
CTIME(FunctionName.of("ctime")),
306+
MKTIME(FunctionName.of("mktime")),
307+
DUR2SEC(FunctionName.of("dur2sec")),
308+
MEMK(FunctionName.of("memk")),
309+
MSTIME(FunctionName.of("mstime")),
310+
RMUNIT(FunctionName.of("rmunit")),
311+
RMCOMMA(FunctionName.of("rmcomma")),
312+
NONE(FunctionName.of("none")),
313+
302314
/** Data Type Convert Function. */
303315
CAST_TO_STRING(FunctionName.of("cast_to_string")),
304316
CAST_TO_BYTE(FunctionName.of("cast_to_byte")),

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,17 @@
6262
import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl;
6363
import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl;
6464
import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
65+
import org.opensearch.sql.expression.function.udf.AutoConvertFunction;
6566
import org.opensearch.sql.expression.function.udf.CryptographicFunction;
67+
import org.opensearch.sql.expression.function.udf.NoneConvertFunction;
68+
import org.opensearch.sql.expression.function.udf.NumConvertFunction;
6669
import org.opensearch.sql.expression.function.udf.ParseFunction;
6770
import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction;
6871
import org.opensearch.sql.expression.function.udf.RexExtractFunction;
6972
import org.opensearch.sql.expression.function.udf.RexExtractMultiFunction;
7073
import org.opensearch.sql.expression.function.udf.RexOffsetFunction;
74+
import org.opensearch.sql.expression.function.udf.RmcommaConvertFunction;
75+
import org.opensearch.sql.expression.function.udf.RmunitConvertFunction;
7176
import org.opensearch.sql.expression.function.udf.SpanFunction;
7277
import org.opensearch.sql.expression.function.udf.ToNumberFunction;
7378
import org.opensearch.sql.expression.function.udf.ToStringFunction;
@@ -419,6 +424,14 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
419424
new NumberToStringFunction().toUDF("NUMBER_TO_STRING");
420425
public static final SqlOperator TONUMBER = new ToNumberFunction().toUDF("TONUMBER");
421426
public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING");
427+
428+
// PPL Convert command functions
429+
public static final SqlOperator AUTO = new AutoConvertFunction().toUDF("AUTO");
430+
public static final SqlOperator NUM = new NumConvertFunction().toUDF("NUM");
431+
public static final SqlOperator RMCOMMA = new RmcommaConvertFunction().toUDF("RMCOMMA");
432+
public static final SqlOperator RMUNIT = new RmunitConvertFunction().toUDF("RMUNIT");
433+
public static final SqlOperator NONE = new NoneConvertFunction().toUDF("NONE");
434+
422435
public static final SqlOperator WIDTH_BUCKET =
423436
new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction()
424437
.toUDF("WIDTH_BUCKET");

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASIN;
2323
import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN;
2424
import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN2;
25+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.AUTO;
2526
import static org.opensearch.sql.expression.function.BuiltinFunctionName.AVG;
2627
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CBRT;
2728
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEIL;
@@ -157,10 +158,12 @@
157158
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVJOIN;
158159
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVMAP;
159160
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVZIP;
161+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NONE;
160162
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT;
161163
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL;
162164
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOW;
163165
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NULLIF;
166+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NUM;
164167
import static org.opensearch.sql.expression.function.BuiltinFunctionName.OR;
165168
import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERCENTILE_APPROX;
166169
import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERIOD_ADD;
@@ -184,6 +187,8 @@
184187
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REX_OFFSET;
185188
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RIGHT;
186189
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RINT;
190+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RMCOMMA;
191+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RMUNIT;
187192
import static org.opensearch.sql.expression.function.BuiltinFunctionName.ROUND;
188193
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RTRIM;
189194
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SCALAR_MAX;
@@ -982,6 +987,14 @@ void populate() {
982987
registerOperator(INTERNAL_PATTERN_PARSER, PPLBuiltinOperators.PATTERN_PARSER);
983988
registerOperator(TONUMBER, PPLBuiltinOperators.TONUMBER);
984989
registerOperator(TOSTRING, PPLBuiltinOperators.TOSTRING);
990+
991+
// Register PPL Convert command functions
992+
registerOperator(AUTO, PPLBuiltinOperators.AUTO);
993+
registerOperator(NUM, PPLBuiltinOperators.NUM);
994+
registerOperator(RMCOMMA, PPLBuiltinOperators.RMCOMMA);
995+
registerOperator(RMUNIT, PPLBuiltinOperators.RMUNIT);
996+
registerOperator(NONE, PPLBuiltinOperators.NONE);
997+
985998
register(
986999
TOSTRING,
9871000
(FunctionImp1)

‎core/src/main/java/org/opensearch/sql/expression/function/udf/AutoConvertFunction.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import java.util.List;
9+
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
10+
import org.apache.calcite.adapter.enumerable.NullPolicy;
11+
import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
12+
import org.apache.calcite.linq4j.tree.Expression;
13+
import org.apache.calcite.linq4j.tree.Expressions;
14+
import org.apache.calcite.rex.RexCall;
15+
import org.apache.calcite.sql.type.ReturnTypes;
16+
import org.apache.calcite.sql.type.SqlReturnTypeInference;
17+
import org.opensearch.sql.calcite.utils.PPLOperandTypes;
18+
import org.opensearch.sql.expression.function.ImplementorUDF;
19+
import org.opensearch.sql.expression.function.UDFOperandMetadata;
20+
21+
/**
22+
* PPL auto() conversion function. Automatically converts string values to numbers using best-fit
23+
* heuristics.
24+
*/
25+
public class AutoConvertFunction extends ImplementorUDF {
26+
27+
public AutoConvertFunction() {
28+
super(new AutoConvertImplementor(), NullPolicy.ANY);
29+
}
30+
31+
@Override
32+
public SqlReturnTypeInference getReturnTypeInference() {
33+
return ReturnTypes.DOUBLE_FORCE_NULLABLE;
34+
}
35+
36+
@Override
37+
public UDFOperandMetadata getOperandMetadata() {
38+
return PPLOperandTypes.OPTIONAL_ANY;
39+
}
40+
41+
public static class AutoConvertImplementor implements NotNullImplementor {
42+
@Override
43+
public Expression implement(
44+
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
45+
Expression fieldValue = translatedOperands.get(0);
46+
return Expressions.call(ConversionUtils.class, "autoConvert", fieldValue);
47+
}
48+
}
49+
}

0 commit comments

Comments
 (0)