Skip to content

Commit 99707a3

Browse files
dvjyothsnasohami
dvjyothsna
authored and
sohami
committed
DRILL-7199: Optimize population of metadata for non-interesting columns
closes #1771
1 parent 7e0e583 commit 99707a3

8 files changed

Lines changed: 165 additions & 23 deletions

File tree

‎exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.apache.drill.metastore.ColumnStatisticsKind;
5252
import org.apache.drill.metastore.FileMetadata;
5353
import org.apache.drill.metastore.LocationProvider;
54+
import org.apache.drill.metastore.NonInterestingColumnsMetadata;
5455
import org.apache.drill.metastore.PartitionMetadata;
5556
import org.apache.drill.metastore.TableMetadata;
5657
import org.apache.drill.metastore.TableStatisticsKind;
@@ -86,6 +87,7 @@ public abstract class AbstractGroupScanWithMetadata extends AbstractFileGroupSca
8687
// partition metadata info: mixed partition values for all partition keys in the same list
8788
protected List<PartitionMetadata> partitions;
8889

90+
protected NonInterestingColumnsMetadata nonInterestingColumnsMetadata;
8991
protected List<SchemaPath> partitionColumns;
9092
protected LogicalExpression filter;
9193
protected List<SchemaPath> columns;
@@ -115,7 +117,7 @@ protected AbstractGroupScanWithMetadata(AbstractGroupScanWithMetadata that) {
115117
this.partitionColumns = that.partitionColumns;
116118
this.partitions = that.partitions;
117119
this.files = that.files;
118-
120+
this.nonInterestingColumnsMetadata = that.nonInterestingColumnsMetadata;
119121
this.fileSet = that.fileSet == null ? null : new HashSet<>(that.fileSet);
120122
}
121123

@@ -151,18 +153,29 @@ public boolean isMatchAllMetadata() {
151153
*/
152154
@Override
153155
public long getColumnValueCount(SchemaPath column) {
154-
long tableRowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(getTableMetadata());
156+
long tableRowCount, colNulls;
157+
Long nulls;
155158
ColumnStatistics columnStats = getTableMetadata().getColumnStatistics(column);
156-
long colNulls;
159+
ColumnStatistics nonInterestingColStats = null;
160+
if (columnStats == null) {
161+
nonInterestingColStats = getNonInterestingColumnsMetadata().getColumnStatistics(column);
162+
}
163+
157164
if (columnStats != null) {
158-
Long nulls = (Long) columnStats.getStatistic(ColumnStatisticsKind.NULLS_COUNT);
159-
colNulls = nulls != null ? nulls : Statistic.NO_COLUMN_STATS;
165+
tableRowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(getTableMetadata());
166+
} else if (nonInterestingColStats != null) {
167+
tableRowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(getNonInterestingColumnsMetadata());
160168
} else {
161-
return 0;
169+
return 0; // returns 0 if the column doesn't exist in the table.
162170
}
171+
172+
columnStats = columnStats != null ? columnStats : nonInterestingColStats;
173+
nulls = (Long) columnStats.getStatistic(ColumnStatisticsKind.NULLS_COUNT);
174+
colNulls = nulls != null ? nulls : Statistic.NO_COLUMN_STATS;
175+
163176
return Statistic.NO_COLUMN_STATS == tableRowCount
164-
|| Statistic.NO_COLUMN_STATS == colNulls
165-
? Statistic.NO_COLUMN_STATS : tableRowCount - colNulls;
177+
|| Statistic.NO_COLUMN_STATS == colNulls
178+
? Statistic.NO_COLUMN_STATS : tableRowCount - colNulls;
166179
}
167180

168181
@Override
@@ -266,6 +279,7 @@ public AbstractGroupScanWithMetadata applyFilter(LogicalExpression filterExpr, U
266279
filteredMetadata.withTable(getTableMetadata())
267280
.withPartitions(getNextOrEmpty(getPartitionsMetadata()))
268281
.withFiles(filesMap)
282+
.withNonInterestingColumns(getNonInterestingColumnsMetadata())
269283
.withMatching(false);
270284
}
271285

@@ -387,6 +401,7 @@ public GroupScan applyLimit(int maxRecords) {
387401
.withTable(getTableMetadata())
388402
.withPartitions(getPartitionsMetadata())
389403
.withFiles(filesMap)
404+
.withNonInterestingColumns(getNonInterestingColumnsMetadata())
390405
.withMatching(matchAllMetadata)
391406
.build();
392407
}
@@ -520,6 +535,14 @@ protected List<PartitionMetadata> getPartitionsMetadata() {
520535
return partitions;
521536
}
522537

538+
@JsonIgnore
539+
public NonInterestingColumnsMetadata getNonInterestingColumnsMetadata() {
540+
if (nonInterestingColumnsMetadata == null) {
541+
nonInterestingColumnsMetadata = metadataProvider.getNonInterestingColumnsMeta();
542+
}
543+
return nonInterestingColumnsMetadata;
544+
}
545+
523546
/**
524547
* This class is responsible for filtering different metadata levels.
525548
*/
@@ -531,6 +554,7 @@ protected abstract static class GroupScanWithMetadataFilterer {
531554
protected TableMetadata tableMetadata;
532555
protected List<PartitionMetadata> partitions = Collections.emptyList();
533556
protected Map<Path, FileMetadata> files = Collections.emptyMap();
557+
protected NonInterestingColumnsMetadata nonInterestingColumnsMetadata;
534558

535559
// for the case when filtering is possible for partitions, but files count exceeds
536560
// PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD, new group scan with at least filtered partitions
@@ -558,6 +582,11 @@ public GroupScanWithMetadataFilterer withPartitions(List<PartitionMetadata> part
558582
return this;
559583
}
560584

585+
public GroupScanWithMetadataFilterer withNonInterestingColumns(NonInterestingColumnsMetadata nonInterestingColumns) {
586+
this.nonInterestingColumnsMetadata = nonInterestingColumns;
587+
return this;
588+
}
589+
561590
public GroupScanWithMetadataFilterer withFiles(Map<Path, FileMetadata> files) {
562591
this.files = files;
563592
return this;
@@ -729,6 +758,9 @@ public <T extends BaseMetadata> List<T> filterAndGetMetadata(Set<SchemaPath> sch
729758
locationProvider.getLocation(), source.supportsFileImplicitColumns());
730759
}
731760

761+
if (source.getNonInterestingColumnsMetadata() != null) {
762+
columnsStatistics.putAll(source.getNonInterestingColumnsMetadata().getColumnsStatistics());
763+
}
732764
RowsMatch match = FilterEvaluatorUtils.matches(filterPredicate,
733765
columnsStatistics, (long) metadata.getStatistic(TableStatisticsKind.ROW_COUNT),
734766
metadata.getSchema(), schemaPathsInExpr);

‎exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.drill.metastore.ColumnStatisticsImpl;
2727
import org.apache.drill.metastore.FileMetadata;
2828
import org.apache.drill.metastore.FileTableMetadata;
29+
import org.apache.drill.metastore.NonInterestingColumnsMetadata;
2930
import org.apache.drill.metastore.PartitionMetadata;
3031
import org.apache.drill.metastore.TableMetadata;
3132
import org.apache.hadoop.fs.Path;
@@ -86,6 +87,11 @@ public List<FileMetadata> getFilesForPartition(PartitionMetadata partition) {
8687
return null;
8788
}
8889

90+
@Override
91+
public NonInterestingColumnsMetadata getNonInterestingColumnsMeta() {
92+
return null;
93+
}
94+
8995
public static class Builder implements SimpleFileTableMetadataProviderBuilder {
9096
private String tableName;
9197
private Path location;

‎exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,7 @@ public AbstractGroupScanWithMetadata applyFilter(LogicalExpression filterExpr, U
290290
builder.withRowGroups(rowGroupsMap)
291291
.withTable(getTableMetadata())
292292
.withPartitions(getNextOrEmpty(getPartitionsMetadata()))
293+
.withNonInterestingColumns(getNonInterestingColumnsMetadata())
293294
.withFiles(filesMap)
294295
.withMatching(false);
295296
}
@@ -363,6 +364,7 @@ public GroupScan applyLimit(int maxRecords) {
363364
.withTable(getTableMetadata())
364365
.withPartitions(getPartitionsMetadata())
365366
.withFiles(qualifiedFiles)
367+
.withNonInterestingColumns(getNonInterestingColumnsMetadata())
366368
.withMatching(matchAllMetadata)
367369
.build();
368370
}
@@ -500,6 +502,7 @@ public AbstractParquetGroupScan build() {
500502
newScan.files = files;
501503
newScan.rowGroups = rowGroups;
502504
newScan.matchAllMetadata = matchAllMetadata;
505+
newScan.nonInterestingColumnsMetadata = nonInterestingColumnsMetadata;
503506
// since builder is used when pruning happens, entries and fileSet should be expanded
504507
if (!newScan.getFilesMetadata().isEmpty()) {
505508
newScan.entries = newScan.getFilesMetadata().keySet().stream()

‎exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.drill.exec.record.metadata.TupleMetadata;
2525
import org.apache.drill.metastore.BaseMetadata;
2626
import org.apache.drill.metastore.ColumnStatisticsImpl;
27+
import org.apache.drill.metastore.NonInterestingColumnsMetadata;
2728
import org.apache.drill.metastore.StatisticsKind;
2829
import org.apache.drill.metastore.TableMetadata;
2930
import org.apache.drill.metastore.TableStatisticsKind;
@@ -88,6 +89,7 @@ public abstract class BaseParquetMetadataProvider implements ParquetMetadataProv
8889
private TableMetadata tableMetadata;
8990
private List<PartitionMetadata> partitions;
9091
private Map<Path, FileMetadata> files;
92+
private NonInterestingColumnsMetadata nonInterestingColumnsMetadata;
9193

9294
// whether metadata for row groups should be collected to create files, partitions and table metadata
9395
private final boolean collectMetadata = false;
@@ -160,6 +162,7 @@ protected void init(BaseParquetMetadataProvider metadataProvider) throws IOExcep
160162
TableMetadata tableMetadata = getTableMetadata();
161163
getPartitionsMetadata();
162164
getRowGroupsMeta();
165+
getNonInterestingColumnsMeta();
163166
this.tableMetadata = ParquetTableMetadataUtils.updateRowCount(tableMetadata, getRowGroupsMeta());
164167
parquetTableMetadata = null;
165168
}
@@ -178,9 +181,18 @@ public void initializeMetadata() throws IOException {
178181
getFilesMetadata();
179182
getPartitionsMetadata();
180183
getRowGroupsMeta();
184+
getNonInterestingColumnsMeta();
181185
parquetTableMetadata = null;
182186
}
183187

188+
@Override
189+
public NonInterestingColumnsMetadata getNonInterestingColumnsMeta() {
190+
if (nonInterestingColumnsMetadata == null) {
191+
nonInterestingColumnsMetadata = ParquetTableMetadataUtils.getNonInterestingColumnsMeta(parquetTableMetadata);
192+
}
193+
return nonInterestingColumnsMetadata;
194+
}
195+
184196
@Override
185197
@SuppressWarnings("unchecked")
186198
public TableMetadata getTableMetadata() {
@@ -235,7 +247,6 @@ public TableMetadata getTableMetadata() {
235247
new ColumnStatisticsImpl(DrillStatsTable.getEstimatedColumnStats(statsTable, column),
236248
ParquetTableMetadataUtils.getNaturalNullsFirstComparator()));
237249
}
238-
columnsStatistics.putAll(ParquetTableMetadataUtils.populateNonInterestingColumnsStats(columnsStatistics.keySet(), parquetTableMetadata));
239250
}
240251
tableMetadata = new FileTableMetadata(tableName, tableLocation, schema, columnsStatistics, tableStatistics,
241252
-1L, "", partitionKeys);

‎exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilterEvaluatorUtils.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.drill.exec.record.metadata.SchemaPathUtils;
2222
import org.apache.drill.exec.record.metadata.TupleMetadata;
2323
import org.apache.drill.exec.store.parquet.metadata.MetadataBase;
24+
import org.apache.drill.metastore.NonInterestingColumnsMetadata;
2425
import org.apache.drill.metastore.RowGroupMetadata;
2526
import org.apache.drill.metastore.TableStatisticsKind;
2627
import org.apache.drill.exec.expr.FilterBuilder;
@@ -62,7 +63,14 @@ public static RowsMatch evalFilter(LogicalExpression expr, MetadataBase.ParquetT
6263
expr.<Set<SchemaPath>, Void, RuntimeException>accept(new FieldReferenceFinder(), null));
6364

6465
RowGroupMetadata rowGroupMetadata = new ArrayList<>(ParquetTableMetadataUtils.getRowGroupsMetadata(footer).values()).get(rowGroupIndex);
66+
NonInterestingColumnsMetadata nonInterestingColumnsMetadata = ParquetTableMetadataUtils.getNonInterestingColumnsMeta(footer);
6567
Map<SchemaPath, ColumnStatistics> columnsStatistics = rowGroupMetadata.getColumnsStatistics();
68+
69+
// Add column statistics of non-interesting columns if there are any
70+
if (nonInterestingColumnsMetadata != null) {
71+
columnsStatistics.putAll(nonInterestingColumnsMetadata.getColumnsStatistics());
72+
}
73+
6674
columnsStatistics = ParquetTableMetadataUtils.addImplicitColumnsStatistics(columnsStatistics,
6775
schemaPathsInExpr, Collections.emptyList(), options, rowGroupMetadata.getLocation(), true);
6876

‎exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.drill.metastore.ColumnStatisticsImpl;
3636
import org.apache.drill.metastore.ColumnStatisticsKind;
3737
import org.apache.drill.metastore.FileMetadata;
38+
import org.apache.drill.metastore.NonInterestingColumnsMetadata;
3839
import org.apache.drill.metastore.PartitionMetadata;
3940
import org.apache.drill.metastore.RowGroupMetadata;
4041
import org.apache.drill.metastore.StatisticsKind;
@@ -195,7 +196,6 @@ public static <T extends BaseMetadata> Map<SchemaPath, ColumnStatistics> mergeCo
195196
}
196197
columnsStatistics.put(column, new ColumnStatisticsImpl(statisticsMap, statisticsList.iterator().next().getValueComparator()));
197198
}
198-
columnsStatistics.putAll(populateNonInterestingColumnsStats(columnsStatistics.keySet(), parquetTableMetadata));
199199
return columnsStatistics;
200200
}
201201

@@ -287,27 +287,27 @@ public static Map<SchemaPath, ColumnStatistics> getRowGroupColumnStatistics(
287287
statistics.put(ColumnStatisticsKind.NULLS_COUNT, nulls);
288288
columnsStatistics.put(colPath, new ColumnStatisticsImpl(statistics, comparator));
289289
}
290-
columnsStatistics.putAll(populateNonInterestingColumnsStats(columnsStatistics.keySet(), tableMetadata));
291290
return columnsStatistics;
292291
}
293292

294293
/**
295-
* Populates the non-interesting column's statistics
296-
* @param schemaPaths columns paths which should be ignored
294+
* Returns the non-interesting column's metadata
297295
* @param parquetTableMetadata the source of column metadata for non-interesting column's statistics
298-
* @return returns non-interesting column statistics map
296+
* @return returns non-interesting columns metadata
299297
*/
300-
@SuppressWarnings("unchecked")
301-
public static Map<SchemaPath, ColumnStatistics> populateNonInterestingColumnsStats(
302-
Set<SchemaPath> schemaPaths, MetadataBase.ParquetTableMetadataBase parquetTableMetadata) {
298+
public static NonInterestingColumnsMetadata getNonInterestingColumnsMeta(MetadataBase.ParquetTableMetadataBase parquetTableMetadata) {
303299
Map<SchemaPath, ColumnStatistics> columnsStatistics = new HashMap<>();
304300
if (parquetTableMetadata instanceof Metadata_V4.ParquetTableMetadata_v4) {
305-
ConcurrentHashMap<Metadata_V4.ColumnTypeMetadata_v4.Key, Metadata_V4.ColumnTypeMetadata_v4 > columnTypeInfoMap =
306-
((Metadata_V4.ParquetTableMetadata_v4) parquetTableMetadata).getColumnTypeInfoMap();
307-
if ( columnTypeInfoMap == null ) { return columnsStatistics; } // in some cases for runtime pruning
301+
ConcurrentHashMap<Metadata_V4.ColumnTypeMetadata_v4.Key, Metadata_V4.ColumnTypeMetadata_v4> columnTypeInfoMap =
302+
((Metadata_V4.ParquetTableMetadata_v4) parquetTableMetadata).getColumnTypeInfoMap();
303+
304+
if (columnTypeInfoMap == null) {
305+
return new NonInterestingColumnsMetadata(columnsStatistics);
306+
} // in some cases for runtime pruning
307+
308308
for (Metadata_V4.ColumnTypeMetadata_v4 columnTypeMetadata : columnTypeInfoMap.values()) {
309-
SchemaPath schemaPath = SchemaPath.getCompoundPath(columnTypeMetadata.name);
310-
if (!schemaPaths.contains(schemaPath)) {
309+
if (!columnTypeMetadata.isInteresting) {
310+
SchemaPath schemaPath = SchemaPath.getCompoundPath(columnTypeMetadata.name);
311311
Map<StatisticsKind, Object> statistics = new HashMap<>();
312312
statistics.put(ColumnStatisticsKind.NULLS_COUNT, Statistic.NO_COLUMN_STATS);
313313
PrimitiveType.PrimitiveTypeName primitiveType = columnTypeMetadata.primitiveType;
@@ -316,8 +316,9 @@ public static Map<SchemaPath, ColumnStatistics> populateNonInterestingColumnsSta
316316
columnsStatistics.put(schemaPath, new ColumnStatisticsImpl<>(statistics, comparator));
317317
}
318318
}
319+
return new NonInterestingColumnsMetadata(columnsStatistics);
319320
}
320-
return columnsStatistics;
321+
return new NonInterestingColumnsMetadata(columnsStatistics);
321322
}
322323

323324
/**

‎metastore/file-metadata/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProvider.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.drill.common.expression.SchemaPath;
2121
import org.apache.drill.metastore.FileMetadata;
22+
import org.apache.drill.metastore.NonInterestingColumnsMetadata;
2223
import org.apache.drill.metastore.PartitionMetadata;
2324
import org.apache.drill.metastore.TableMetadata;
2425
import org.apache.hadoop.fs.Path;
@@ -85,4 +86,10 @@ public interface TableMetadataProvider {
8586
* @return list of {@link FileMetadata} instances which belongs to specified partitions
8687
*/
8788
List<FileMetadata> getFilesForPartition(PartitionMetadata partition);
89+
90+
/**
91+
* Returns {@link NonInterestingColumnsMetadata} instance which provides metadata for non-interesting columns.
92+
* @return {@link NonInterestingColumnsMetadata} instance
93+
*/
94+
NonInterestingColumnsMetadata getNonInterestingColumnsMeta();
8895
}

0 commit comments

Comments
 (0)