Skip to content

Commit 4a4f890

Browse files
yuancuyuancu
committed
Trying to fix patterns
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
1 parent 2d36650 commit 4a4f890

5 files changed

Lines changed: 64 additions & 41 deletions

File tree

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 32 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3365,15 +3365,21 @@ private void flattenParsedPattern(
33653365
Boolean showNumberedToken) {
33663366
List<RexNode> fattenedNodes = new ArrayList<>();
33673367
List<String> projectNames = new ArrayList<>();
3368+
3369+
// Check if parsedNode is a STRUCT type (ROW) or MAP type
3370+
boolean isStructType = parsedNode.getType().getSqlTypeName() == SqlTypeName.ROW;
3371+
33683372
// Flatten map struct fields
33693373
RexNode patternExpr =
33703374
context.rexBuilder.makeCast(
33713375
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR),
3372-
PPLFuncImpTable.INSTANCE.resolve(
3373-
context.rexBuilder,
3374-
BuiltinFunctionName.INTERNAL_ITEM,
3375-
parsedNode,
3376-
context.rexBuilder.makeLiteral(PatternUtils.PATTERN)),
3376+
isStructType
3377+
? context.rexBuilder.makeFieldAccess(parsedNode, PatternUtils.PATTERN, false)
3378+
: PPLFuncImpTable.INSTANCE.resolve(
3379+
context.rexBuilder,
3380+
BuiltinFunctionName.INTERNAL_ITEM,
3381+
parsedNode,
3382+
context.rexBuilder.makeLiteral(PatternUtils.PATTERN)),
33773383
true,
33783384
true);
33793385
fattenedNodes.add(context.relBuilder.alias(patternExpr, originalPatternResultAlias));
@@ -3382,11 +3388,13 @@ private void flattenParsedPattern(
33823388
RexNode patternCountExpr =
33833389
context.rexBuilder.makeCast(
33843390
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
3385-
PPLFuncImpTable.INSTANCE.resolve(
3386-
context.rexBuilder,
3387-
BuiltinFunctionName.INTERNAL_ITEM,
3388-
parsedNode,
3389-
context.rexBuilder.makeLiteral(PatternUtils.PATTERN_COUNT)),
3391+
isStructType
3392+
? context.rexBuilder.makeFieldAccess(parsedNode, PatternUtils.PATTERN_COUNT, false)
3393+
: PPLFuncImpTable.INSTANCE.resolve(
3394+
context.rexBuilder,
3395+
BuiltinFunctionName.INTERNAL_ITEM,
3396+
parsedNode,
3397+
context.rexBuilder.makeLiteral(PatternUtils.PATTERN_COUNT)),
33903398
true,
33913399
true);
33923400
fattenedNodes.add(context.relBuilder.alias(patternCountExpr, PatternUtils.PATTERN_COUNT));
@@ -3396,11 +3404,13 @@ private void flattenParsedPattern(
33963404
RexNode tokensExpr =
33973405
context.rexBuilder.makeCast(
33983406
UserDefinedFunctionUtils.tokensMap,
3399-
PPLFuncImpTable.INSTANCE.resolve(
3400-
context.rexBuilder,
3401-
BuiltinFunctionName.INTERNAL_ITEM,
3402-
parsedNode,
3403-
context.rexBuilder.makeLiteral(PatternUtils.TOKENS)),
3407+
isStructType
3408+
? context.rexBuilder.makeFieldAccess(parsedNode, PatternUtils.TOKENS, false)
3409+
: PPLFuncImpTable.INSTANCE.resolve(
3410+
context.rexBuilder,
3411+
BuiltinFunctionName.INTERNAL_ITEM,
3412+
parsedNode,
3413+
context.rexBuilder.makeLiteral(PatternUtils.TOKENS)),
34043414
true,
34053415
true);
34063416
fattenedNodes.add(context.relBuilder.alias(tokensExpr, PatternUtils.TOKENS));
@@ -3414,11 +3424,13 @@ private void flattenParsedPattern(
34143424
.getTypeFactory()
34153425
.createArrayType(
34163426
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), -1),
3417-
PPLFuncImpTable.INSTANCE.resolve(
3418-
context.rexBuilder,
3419-
BuiltinFunctionName.INTERNAL_ITEM,
3420-
explicitMapType(context, parsedNode, SqlTypeName.VARCHAR),
3421-
context.rexBuilder.makeLiteral(PatternUtils.SAMPLE_LOGS)),
3427+
isStructType
3428+
? context.rexBuilder.makeFieldAccess(parsedNode, PatternUtils.SAMPLE_LOGS, false)
3429+
: PPLFuncImpTable.INSTANCE.resolve(
3430+
context.rexBuilder,
3431+
BuiltinFunctionName.INTERNAL_ITEM,
3432+
explicitMapType(context, parsedNode, SqlTypeName.VARCHAR),
3433+
context.rexBuilder.makeLiteral(PatternUtils.SAMPLE_LOGS)),
34223434
true,
34233435
true);
34243436
fattenedNodes.add(context.relBuilder.alias(sampleLogsExpr, PatternUtils.SAMPLE_LOGS));

‎core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
package org.opensearch.sql.calcite.utils;
77

88
import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType;
9-
import static org.apache.calcite.sql.type.SqlTypeUtil.createMapType;
109
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.*;
1110
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.*;
1211

@@ -57,24 +56,35 @@ public class UserDefinedFunctionUtils {
5756
public static final RelDataType NULLABLE_STRING =
5857
TYPE_FACTORY.createTypeWithNullability(TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR), true);
5958
public static final RelDataType NULLABLE_IP_UDT = TYPE_FACTORY.createUDT(EXPR_IP, true);
60-
61-
public static RelDataType nullablePatternAggList =
62-
createArrayType(
63-
TYPE_FACTORY,
64-
TYPE_FACTORY.createMapType(
65-
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
66-
TYPE_FACTORY.createSqlType(SqlTypeName.ANY)),
67-
true);
68-
public static RelDataType patternStruct =
69-
createMapType(
70-
TYPE_FACTORY,
71-
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
72-
TYPE_FACTORY.createSqlType(SqlTypeName.ANY),
73-
false);
7459
public static RelDataType tokensMap =
7560
TYPE_FACTORY.createMapType(
7661
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
7762
createArrayType(TYPE_FACTORY, TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR), false));
63+
public static RelDataType patternElementStruct =
64+
TYPE_FACTORY.createStructType(
65+
List.of(
66+
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
67+
TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT),
68+
tokensMap,
69+
createArrayType(
70+
TYPE_FACTORY,
71+
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
72+
false),
73+
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR)),
74+
List.of("pattern", "pattern_count", "tokens", "sample_logs", "patterns_field"));
75+
public static RelDataType nullablePatternAggList =
76+
createArrayType(TYPE_FACTORY, patternElementStruct, true);
77+
public static RelDataType patternStruct =
78+
TYPE_FACTORY.createStructType(
79+
List.of(
80+
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR), // pattern
81+
TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT), // pattern_count
82+
tokensMap, // tokens
83+
createArrayType(
84+
TYPE_FACTORY,
85+
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
86+
false)), // sample_logs),
87+
List.of("pattern", "pattern_count", "tokens", "sample_logs"));
7888
public static Set<String> SINGLE_FIELD_RELEVANCE_FUNCTION_SET =
7989
ImmutableSet.of("match", "match_phrase", "match_bool_prefix", "match_phrase_prefix");
8090
public static Set<String> MULTI_FIELDS_RELEVANCE_FUNCTION_SET =

‎core/src/main/java/org/opensearch/sql/executor/QueryService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,8 @@ public void executeWithCalcite(
128128
buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType);
129129
RelNode relNode = analyze(plan, context);
130130
relNode = mergeAdjacentFilters(relNode);
131-
RelNode validated = validate(relNode, context);
132-
RelNode optimized = optimize(validated, context);
131+
// RelNode validated = validate(relNode, context);
132+
RelNode optimized = optimize(relNode, context);
133133
RelNode calcitePlan = convertToCalcitePlan(optimized);
134134
executionEngine.execute(calcitePlan, context, listener);
135135
} catch (Throwable t) {

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
2222
import org.apache.calcite.avatica.util.TimeUnit;
2323
import org.apache.calcite.linq4j.tree.Expression;
24+
import org.apache.calcite.rel.type.RelDataType;
2425
import org.apache.calcite.rex.RexCall;
2526
import org.apache.calcite.sql.SqlAggFunction;
2627
import org.apache.calcite.sql.SqlBasicCall;
@@ -475,7 +476,9 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
475476
createUserDefinedAggFunction(
476477
LogPatternAggFunction.class,
477478
"pattern",
478-
ReturnTypes.explicit(UserDefinedFunctionUtils.nullablePatternAggList),
479+
(cx) -> {
480+
List<RelDataType> types = cx.collectOperandTypes();
481+
return UserDefinedFunctionUtils.patternStruct;},
479482
UDFOperandMetadata.wrap(
480483
OperandTypes.VARIADIC)); // operand types of patterns are very flexible
481484
public static final SqlAggFunction LIST =

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1075,9 +1075,7 @@ void populate() {
10751075
// of all operands.
10761076
registerOperator(
10771077
INTERNAL_ITEM,
1078-
SqlStdOperatorTable.ITEM,
1079-
OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER)
1080-
.or(OperandTypes.family(SqlTypeFamily.MAP, SqlTypeFamily.ANY)));
1078+
SqlStdOperatorTable.ITEM);
10811079
registerOperator(
10821080
XOR,
10831081
SqlStdOperatorTable.NOT_EQUALS,

0 commit comments

Comments
 (0)