Skip to content

Commit 351730d

Browse files
LantaoJinLantaoJin
committed
Disable the collapse pushdown
Signed-off-by: Lantao Jin <ltjin@amazon.com>
1 parent 625d241 commit 351730d

8 files changed

Lines changed: 77 additions & 85 deletions

File tree

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -840,13 +840,13 @@ public RelNode visitJoin(Join node, CalcitePlanContext context) {
840840
if (node.getJoinFields().isPresent()) {
841841
joinCondition =
842842
node.getJoinFields().get().stream()
843-
.map(field -> buildJoinConditionByFieldNames(context, field.getField().toString()))
843+
.map(field -> buildJoinConditionByFieldName(context, field.getField().toString()))
844844
.reduce(context.rexBuilder::and)
845845
.orElse(context.relBuilder.literal(true));
846846
} else {
847847
joinCondition =
848848
duplicatedFieldNames.stream()
849-
.map(fieldName -> buildJoinConditionByFieldNames(context, fieldName))
849+
.map(fieldName -> buildJoinConditionByFieldName(context, fieldName))
850850
.reduce(context.rexBuilder::and)
851851
.orElse(context.relBuilder.literal(true));
852852
}
@@ -974,7 +974,7 @@ public Void visitInputRef(RexInputRef inputRef) {
974974
.toList();
975975
}
976976

977-
private static RexNode buildJoinConditionByFieldNames(
977+
private static RexNode buildJoinConditionByFieldName(
978978
CalcitePlanContext context, String fieldName) {
979979
RexNode lookupKey = JoinAndLookupUtils.analyzeFieldsForLookUp(fieldName, false, context);
980980
RexNode sourceKey = JoinAndLookupUtils.analyzeFieldsForLookUp(fieldName, true, context);

‎docs/user/admin/settings.rst‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -792,4 +792,16 @@ If Calcite pushdown optimization is enabled, this setting is used to estimate th
792792

793793
1. The default value is 0.9 since 3.1.0.
794794
2. This setting is node scope.
795-
3. This setting can be updated dynamically.
795+
3. This setting can be updated dynamically.
796+
797+
plugins.calcite.all_join_types.allowed
798+
======================================
799+
800+
Description
801+
-----------
802+
803+
Join types ``inner``, ``left``, ``outer`` (alias of ``left``), ``semi`` and ``anti`` are supported by default. ``right``, ``full``, ``cross`` are performance sensitive join types which are disabled by default. Set config ``plugins.calcite.all_join_types.allowed = true`` to enable.
804+
805+
1. The default value is false since 3.3.0.
806+
2. This setting is node scope.
807+
3. This setting can be updated dynamically.

‎docs/user/ppl/admin/settings.rst‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,3 +159,33 @@ Rollback to default value::
159159
}
160160

161161
Note: the legacy settings of ``opendistro.query.size_limit`` is deprecated, it will fallback to the new settings if you request an update with the legacy name.
162+
163+
plugins.calcite.all_join_types.allowed
164+
======================================
165+
166+
Description
167+
-----------
168+
169+
Since 3.3.0, join types ``inner``, ``left``, ``outer`` (alias of ``left``), ``semi`` and ``anti`` are supported by default. ``right``, ``full``, ``cross`` are performance sensitive join types which are disabled by default. Set config ``plugins.calcite.all_join_types.allowed = true`` to enable.
170+
171+
Example
172+
-------
173+
174+
PPL query::
175+
176+
sh$ curl -sS -H 'Content-Type: application/json' \
177+
... -X PUT localhost:9200/_plugins/_query/settings \
178+
... -d '{"transient" : {"plugins.calcite.all_join_types.allowed" : "true"}}'
179+
{
180+
"acknowledged": true,
181+
"persistent": {},
182+
"transient": {
183+
"plugins": {
184+
"calcite": {
185+
"all_join_types": {
186+
"allowed": "true"
187+
}
188+
}
189+
}
190+
}
191+
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public void supportSearchSargPushDown_timeRange() throws IOException {
6868
}
6969

7070
// Only for Calcite
71-
@Test
71+
@Ignore("https://github.com/opensearch-project/OpenSearch/issues/3725")
7272
public void testJoinWithCriteriaAndMaxOption() throws IOException {
7373
String query =
7474
"source=opensearch-sql_test_index_bank | join max=1 left=l right=r on"
@@ -79,7 +79,7 @@ public void testJoinWithCriteriaAndMaxOption() throws IOException {
7979
}
8080

8181
// Only for Calcite
82-
@Test
82+
@Ignore("https://github.com/opensearch-project/OpenSearch/issues/3725")
8383
public void testJoinWithFieldListAndMaxOption() throws IOException {
8484
String query =
8585
"source=opensearch-sql_test_index_bank | join type=inner max=1 account_number"

‎integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,7 @@ public void testStatsByTimeSpan() throws IOException {
468468
String.format("source=%s | stats count() by span(birthdate,1M)", TEST_INDEX_BANK)));
469469
}
470470

471-
@Test
471+
@Ignore("https://github.com/opensearch-project/OpenSearch/issues/3725")
472472
public void testDedupPushdown() throws IOException {
473473
String expected = loadExpectedPlan("explain_dedup_push.json");
474474
assertJsonEqualsIgnoreId(
@@ -488,7 +488,7 @@ public void testDedupKeepEmptyTruePushdown() throws IOException {
488488
+ " | dedup gender KEEPEMPTY=true"));
489489
}
490490

491-
@Test
491+
@Ignore("https://github.com/opensearch-project/OpenSearch/issues/3725")
492492
public void testDedupKeepEmptyFalsePushdown() throws IOException {
493493
String expected = loadExpectedPlan("explain_dedup_keepempty_false_push.json");
494494
assertJsonEqualsIgnoreId(

‎opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexRules.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ public class OpenSearchIndexRules {
3737
COUNT_STAR_INDEX_SCAN,
3838
LIMIT_INDEX_SCAN,
3939
SORT_INDEX_SCAN,
40-
DEDUP_PUSH_DOWN,
40+
// TODO enable if https://github.com/opensearch-project/OpenSearch/issues/3725 resolved
41+
// DEDUP_PUSH_DOWN,
4142
SORT_PROJECT_EXPR_TRANSPOSE,
4243
EXPAND_COLLATION_ON_PROJECT_EXPR);
4344

‎ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java‎

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -6,39 +6,6 @@
66
package org.opensearch.sql.ppl.parser;
77

88
import static org.opensearch.sql.expression.function.BuiltinFunctionName.*;
9-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinaryArithmeticContext;
10-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext;
11-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BySpanClauseContext;
12-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CompareExprContext;
13-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ConvertedDataTypeContext;
14-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CountAllFunctionCallContext;
15-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CountEvalFunctionCallContext;
16-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DataTypeFunctionCallContext;
17-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DecimalLiteralContext;
18-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DistinctCountFunctionCallContext;
19-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DoubleLiteralContext;
20-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalClauseContext;
21-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalFunctionCallContext;
22-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldExpressionContext;
23-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FloatLiteralContext;
24-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IdentsAsQualifiedNameContext;
25-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IdentsAsTableQualifiedNameContext;
26-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IdentsAsWildcardQualifiedNameContext;
27-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.InExprContext;
28-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IntegerLiteralContext;
29-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IntervalLiteralContext;
30-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalAndContext;
31-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalNotContext;
32-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalOrContext;
33-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalXorContext;
34-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.MultiFieldRelevanceFunctionContext;
35-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SingleFieldRelevanceFunctionContext;
36-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SortFieldContext;
37-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SpanClauseContext;
38-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StatsFunctionCallContext;
39-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StringLiteralContext;
40-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TableSourceContext;
41-
import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.WcFieldExpressionContext;
429

4310
import com.google.common.collect.ImmutableList;
4411
import com.google.common.collect.ImmutableMap;
@@ -59,6 +26,7 @@
5926
import org.opensearch.sql.ast.expression.Alias;
6027
import org.opensearch.sql.ast.expression.AllFields;
6128
import org.opensearch.sql.ast.expression.And;
29+
import org.opensearch.sql.ast.expression.Argument;
6230
import org.opensearch.sql.ast.expression.Between;
6331
import org.opensearch.sql.ast.expression.Case;
6432
import org.opensearch.sql.ast.expression.Cast;
@@ -97,6 +65,7 @@
9765
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CompareExprContext;
9866
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ConvertedDataTypeContext;
9967
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CountAllFunctionCallContext;
68+
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CountEvalFunctionCallContext;
10069
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DataTypeFunctionCallContext;
10170
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DecimalLiteralContext;
10271
import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DistinctCountFunctionCallContext;

‎ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java‎

Lines changed: 23 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -168,53 +168,33 @@ public static Argument getArgumentValue(OpenSearchPPLParser.JoinTypeContext ctx)
168168
}
169169

170170
public static Join.JoinType getJoinType(OpenSearchPPLParser.SqlLikeJoinTypeContext ctx) {
171-
Join.JoinType joinType;
172-
if (ctx == null) {
173-
joinType = Join.JoinType.INNER;
174-
} else if (ctx.INNER() != null) {
175-
joinType = Join.JoinType.INNER;
176-
} else if (ctx.SEMI() != null) {
177-
joinType = Join.JoinType.SEMI;
178-
} else if (ctx.ANTI() != null) {
179-
joinType = Join.JoinType.ANTI;
180-
} else if (ctx.LEFT() != null) {
181-
joinType = Join.JoinType.LEFT;
182-
} else if (ctx.RIGHT() != null) {
183-
joinType = Join.JoinType.RIGHT;
184-
} else if (ctx.CROSS() != null) {
185-
joinType = Join.JoinType.CROSS;
186-
} else if (ctx.FULL() != null) {
187-
joinType = Join.JoinType.FULL;
188-
} else if (ctx.OUTER() != null) {
189-
// type=outer equals type=left outer
190-
joinType = Join.JoinType.LEFT;
191-
} else {
192-
joinType = Join.JoinType.INNER;
171+
try {
172+
if (ctx == null) return Join.JoinType.INNER;
173+
if (ctx.INNER() != null) return Join.JoinType.valueOf(ctx.INNER().getText());
174+
if (ctx.SEMI() != null) return Join.JoinType.valueOf(ctx.SEMI().getText());
175+
if (ctx.ANTI() != null) return Join.JoinType.valueOf(ctx.ANTI().getText());
176+
if (ctx.LEFT() != null) return Join.JoinType.valueOf(ctx.LEFT().getText());
177+
if (ctx.RIGHT() != null) return Join.JoinType.valueOf(ctx.RIGHT().getText());
178+
if (ctx.CROSS() != null) return Join.JoinType.valueOf(ctx.CROSS().getText());
179+
if (ctx.FULL() != null) return Join.JoinType.valueOf(ctx.FULL().getText());
180+
if (ctx.OUTER() != null) return Join.JoinType.LEFT; // Special case
181+
return Join.JoinType.INNER; // Default case
182+
} catch (Exception e) {
183+
throw new SemanticCheckException(String.format("Unsupported join type %s", ctx.getText()));
193184
}
194-
return joinType;
195185
}
196186

197187
public static Join.JoinType getJoinType(OpenSearchPPLParser.JoinTypeContext ctx) {
198-
if (ctx.INNER() != null) {
199-
return Join.JoinType.INNER;
200-
} else if (ctx.SEMI() != null) {
201-
return Join.JoinType.SEMI;
202-
} else if (ctx.ANTI() != null) {
203-
return Join.JoinType.ANTI;
204-
} else if (ctx.LEFT() != null) {
205-
return Join.JoinType.LEFT;
206-
} else if (ctx.RIGHT() != null) {
207-
return Join.JoinType.RIGHT;
208-
} else if (ctx.CROSS() != null) {
209-
return Join.JoinType.CROSS;
210-
} else if (ctx.FULL() != null) {
211-
return Join.JoinType.FULL;
212-
} else if (ctx.OUTER() != null) {
213-
// type=outer equals type=left outer
214-
return Join.JoinType.LEFT;
215-
} else {
216-
throw new SemanticCheckException(String.format("Unsupported join type %s", ctx.getText()));
217-
}
188+
if (ctx == null) return Join.JoinType.INNER;
189+
if (ctx.INNER() != null) return Join.JoinType.INNER;
190+
if (ctx.SEMI() != null) return Join.JoinType.SEMI;
191+
if (ctx.ANTI() != null) return Join.JoinType.ANTI;
192+
if (ctx.LEFT() != null) return Join.JoinType.LEFT;
193+
if (ctx.RIGHT() != null) return Join.JoinType.RIGHT;
194+
if (ctx.CROSS() != null) return Join.JoinType.CROSS;
195+
if (ctx.FULL() != null) return Join.JoinType.FULL;
196+
if (ctx.OUTER() != null) return Join.JoinType.LEFT; // Special case
197+
throw new SemanticCheckException(String.format("Unsupported join type %s", ctx.getText()));
218198
}
219199

220200
public static Join.JoinType getJoinType(Argument.ArgumentMap argumentMap) {

0 commit comments

Comments
 (0)