Skip to content

Commit 93fa20f

Browse files
LantaoJinLantaoJin
authored
Support describe command with Calcite (opensearch-project#3624)
* Support describe command with Calcite Signed-off-by: Lantao Jin <ltjin@amazon.com> * add java doc and class rename Signed-off-by: Lantao Jin <ltjin@amazon.com> * refactor to remove the useless Queryable interface Signed-off-by: Lantao Jin <ltjin@amazon.com> --------- Signed-off-by: Lantao Jin <ltjin@amazon.com>
1 parent 94fb171 commit 93fa20f

19 files changed

Lines changed: 381 additions & 149 deletions

‎core/src/main/java/org/opensearch/sql/calcite/plan/AbstractOpenSearchTable.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.plan;
7+
8+
import org.apache.calcite.rel.type.RelDataType;
9+
import org.apache.calcite.rel.type.RelDataTypeFactory;
10+
import org.apache.calcite.schema.TranslatableTable;
11+
import org.apache.calcite.schema.impl.AbstractTable;
12+
import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory;
13+
14+
/**
15+
* Abstract class to map the {@link org.opensearch.sql.storage.Table} and {@link
16+
* org.apache.calcite.schema.Table}
17+
*/
18+
public abstract class AbstractOpenSearchTable extends AbstractTable
19+
implements TranslatableTable, org.opensearch.sql.storage.Table {
20+
21+
@Override
22+
public RelDataType getRowType(RelDataTypeFactory relDataTypeFactory) {
23+
return OpenSearchTypeFactory.convertSchema(this);
24+
}
25+
}

‎core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchQueryable.java‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchTable.java‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import java.lang.reflect.Type;
3434
import java.nio.charset.Charset;
3535
import java.util.ArrayList;
36+
import java.util.LinkedHashMap;
3637
import java.util.List;
3738
import java.util.Locale;
3839
import java.util.Map;
@@ -308,7 +309,7 @@ public static ExprValue getExprValueByExprType(ExprType type, Object value) {
308309
public static RelDataType convertSchema(Table table) {
309310
List<String> fieldNameList = new ArrayList<>();
310311
List<RelDataType> typeList = new ArrayList<>();
311-
Map<String, ExprType> fieldTypes = table.getFieldTypes();
312+
Map<String, ExprType> fieldTypes = new LinkedHashMap<>(table.getFieldTypes());
312313
fieldTypes.putAll(table.getReservedFieldTypes());
313314
for (Entry<String, ExprType> entry : fieldTypes.entrySet()) {
314315
fieldNameList.add(entry.getKey());

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDescribeCommandIT.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ public class CalciteDescribeCommandIT extends DescribeCommandIT {
1212
public void init() throws Exception {
1313
super.init();
1414
enableCalcite();
15-
// TODO: "https://github.com/opensearch-project/sql/issues/3460"
16-
// disallowCalciteFallback();
15+
disallowCalciteFallback();
1716
}
1817
}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/EnumerableSystemIndexScanRule.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.opensearch.planner.physical;
7+
8+
import org.apache.calcite.adapter.enumerable.EnumerableConvention;
9+
import org.apache.calcite.plan.Convention;
10+
import org.apache.calcite.plan.RelOptRuleCall;
11+
import org.apache.calcite.rel.RelNode;
12+
import org.apache.calcite.rel.convert.ConverterRule;
13+
import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan;
14+
import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan;
15+
16+
/**
17+
* Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link
18+
* CalciteEnumerableSystemIndexScan}.
19+
*/
20+
public class EnumerableSystemIndexScanRule extends ConverterRule {
21+
/** Default configuration. */
22+
public static final Config DEFAULT_CONFIG =
23+
Config.INSTANCE
24+
.as(Config.class)
25+
.withConversion(
26+
CalciteLogicalSystemIndexScan.class,
27+
s -> s.getSysIndex() != null,
28+
Convention.NONE,
29+
EnumerableConvention.INSTANCE,
30+
"EnumerableSystemIndexScanRule")
31+
.withRuleFactory(EnumerableSystemIndexScanRule::new);
32+
33+
/** Creates an EnumerableProjectRule. */
34+
protected EnumerableSystemIndexScanRule(Config config) {
35+
super(config);
36+
}
37+
38+
@Override
39+
public boolean matches(RelOptRuleCall call) {
40+
CalciteLogicalSystemIndexScan scan = call.rel(0);
41+
return scan.getVariablesSet().isEmpty();
42+
}
43+
44+
@Override
45+
public RelNode convert(RelNode rel) {
46+
final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel;
47+
return new CalciteEnumerableSystemIndexScan(
48+
scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema());
49+
}
50+
}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java‎

Lines changed: 2 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,17 @@
88
import com.google.common.annotations.VisibleForTesting;
99
import java.util.HashMap;
1010
import java.util.LinkedHashMap;
11-
import java.util.List;
1211
import java.util.Map;
1312
import java.util.Map.Entry;
1413
import java.util.function.Function;
1514
import java.util.stream.Collectors;
1615
import lombok.Getter;
1716
import lombok.RequiredArgsConstructor;
18-
import org.apache.calcite.linq4j.AbstractEnumerable;
19-
import org.apache.calcite.linq4j.Enumerable;
20-
import org.apache.calcite.linq4j.Enumerator;
2117
import org.apache.calcite.plan.RelOptCluster;
2218
import org.apache.calcite.plan.RelOptTable;
2319
import org.apache.calcite.rel.RelNode;
2420
import org.opensearch.common.unit.TimeValue;
25-
import org.opensearch.sql.calcite.plan.OpenSearchTable;
21+
import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable;
2622
import org.opensearch.sql.common.setting.Settings;
2723
import org.opensearch.sql.data.type.ExprCoreType;
2824
import org.opensearch.sql.data.type.ExprType;
@@ -37,7 +33,6 @@
3733
import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder;
3834
import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest;
3935
import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan;
40-
import org.opensearch.sql.opensearch.storage.scan.OpenSearchIndexEnumerator;
4136
import org.opensearch.sql.opensearch.storage.scan.OpenSearchIndexScan;
4237
import org.opensearch.sql.opensearch.storage.scan.OpenSearchIndexScanBuilder;
4338
import org.opensearch.sql.planner.DefaultImplementor;
@@ -50,7 +45,7 @@
5045
import org.opensearch.sql.storage.read.TableScanBuilder;
5146

5247
/** OpenSearch table (index) implementation. */
53-
public class OpenSearchIndex extends OpenSearchTable {
48+
public class OpenSearchIndex extends AbstractOpenSearchTable {
5449

5550
public static final String METADATA_FIELD_ID = "_id";
5651
public static final String METADATA_FIELD_INDEX = "_index";
@@ -94,7 +89,6 @@ public class OpenSearchIndex extends OpenSearchTable {
9489

9590
/** Constructor. */
9691
public OpenSearchIndex(OpenSearchClient client, Settings settings, String indexName) {
97-
super(null);
9892
this.client = client;
9993
this.settings = settings;
10094
this.indexName = new OpenSearchRequest.IndexName(indexName);
@@ -260,26 +254,6 @@ public PhysicalPlan visitEval(LogicalEval node, OpenSearchIndexScan context) {
260254
}
261255
}
262256

263-
@Override
264-
public Enumerable<Object> search() {
265-
return new AbstractEnumerable<>() {
266-
@Override
267-
public Enumerator<Object> enumerator() {
268-
final int querySizeLimit = settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT);
269-
270-
final TimeValue cursorKeepAlive =
271-
settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE);
272-
var builder =
273-
new OpenSearchRequestBuilder(querySizeLimit, createExprValueFactory(), settings);
274-
return new OpenSearchIndexEnumerator(
275-
client,
276-
List.copyOf(getFieldTypes().keySet()),
277-
builder.getMaxResponseSize(),
278-
builder.build(indexName, getMaxResultWindow(), cursorKeepAlive, client));
279-
}
280-
};
281-
}
282-
283257
public OpenSearchRequestBuilder createRequestBuilder() {
284258
return new OpenSearchRequestBuilder(
285259
settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT),

opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteIndexScan.java renamed to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@
2626
import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder;
2727
import org.opensearch.sql.opensearch.storage.OpenSearchIndex;
2828

29-
/** Relational expression representing a scan of an OpenSearchIndex type. */
29+
/** An abstract relational operator representing a scan of an OpenSearchIndex type. */
3030
@Getter
31-
public abstract class CalciteIndexScan extends TableScan {
31+
public abstract class AbstractCalciteIndexScan extends TableScan {
3232
public final OpenSearchIndex osIndex;
3333
// The schema of this scan operator, it's initialized with the row type of the table, but may be
3434
// changed by push down operations.
@@ -40,7 +40,7 @@ public abstract class CalciteIndexScan extends TableScan {
4040
// so we cannot apply these actions right away.
4141
protected final PushDownContext pushDownContext;
4242

43-
protected CalciteIndexScan(
43+
protected AbstractCalciteIndexScan(
4444
RelOptCluster cluster,
4545
RelTraitSet traitSet,
4646
List<RelHint> hints,

‎opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@
3131
import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder;
3232
import org.opensearch.sql.opensearch.storage.OpenSearchIndex;
3333

34-
/** Relational expression representing a scan of an OpenSearchIndex type. */
35-
public class CalciteEnumerableIndexScan extends CalciteIndexScan implements EnumerableRel {
34+
/** The physical relational operator representing a scan of an OpenSearchIndex type. */
35+
public class CalciteEnumerableIndexScan extends AbstractCalciteIndexScan implements EnumerableRel {
3636
private static final Logger LOG = LogManager.getLogger(CalciteEnumerableIndexScan.class);
3737

3838
/**

‎opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,9 @@
3737
import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser;
3838
import org.opensearch.sql.opensearch.storage.OpenSearchIndex;
3939

40+
/** The logical relational operator representing a scan of an OpenSearchIndex type. */
4041
@Getter
41-
public class CalciteLogicalIndexScan extends CalciteIndexScan {
42+
public class CalciteLogicalIndexScan extends AbstractCalciteIndexScan {
4243
private static final Logger LOG = LogManager.getLogger(CalciteLogicalIndexScan.class);
4344

4445
public CalciteLogicalIndexScan(

0 commit comments

Comments
 (0)