all reading files with no path_in_schema

Author: etseidl

parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java

@@ -608,7 +608,6 @@ private void addRowGroup(
       ColumnMetaData metaData = new ColumnMetaData(
           getType(columnMetaData.getType()),
           toFormatEncodings(columnMetaData.getEncodings()),
-          columnMetaData.getPath().toList(),
           toFormatCodec(columnMetaData.getCodec()),
           columnMetaData.getValueCount(),
           columnMetaData.getTotalUncompressedSize(),
@@ -619,6 +618,9 @@ private void addRowGroup(
           || columnMetaData.hasDictionaryPage()) {
         metaData.setDictionary_page_offset(columnMetaData.getDictionaryPageOffset());
       }
+      if (path != null) {
+        metaData.setPath_in_schema(path.toList());
+      }
       long bloomFilterOffset = columnMetaData.getBloomFilterOffset();
       if (bloomFilterOffset >= 0) {
         metaData.setBloom_filter_offset(bloomFilterOffset);
@@ -1830,6 +1832,9 @@ public ParquetMetadata fromParquetMetadata(
     List<BlockMetaData> blocks = new ArrayList<BlockMetaData>();
     List<RowGroup> row_groups = parquetMetadata.getRow_groups();
 
+    // needed to create path_in_schema for leaf columns
+    List<String[]> col_paths = messageType.getPaths();
+
     if (row_groups != null) {
       for (RowGroup rowGroup : row_groups) {
         BlockMetaData blockMetaData = new BlockMetaData();
@@ -1859,7 +1864,7 @@ public ParquetMetadata fromParquetMetadata(
           boolean lazyMetadataDecryption = false;
 
           if (null == cryptoMetaData) { // Plaintext column
-            columnPath = getPath(metaData);
+            columnPath = ColumnPath.get(col_paths.get(columnOrdinal));
             if (null != fileDecryptor && !fileDecryptor.plaintextFile()) {
               // mark this column as plaintext in encrypted file decryptor
               fileDecryptor.setColumnCryptoMetadata(
@@ -1876,7 +1881,7 @@ public ParquetMetadata fromParquetMetadata(
                 throw new ParquetCryptoRuntimeException(
                     "ColumnMetaData not set in Encryption with Footer key");
               }
-              columnPath = getPath(metaData);
+              columnPath = ColumnPath.get(col_paths.get(columnOrdinal));
               if (!encryptedFooter) { // Unencrypted footer. Decrypt full column metadata, using footer
                 // key
                 ByteArrayInputStream tempInputStream =
@@ -1983,11 +1988,6 @@ private static IndexReference toOffsetIndexReference(ColumnChunk columnChunk) {
     return null;
   }
 
-  private static ColumnPath getPath(ColumnMetaData metaData) {
-    String[] path = metaData.path_in_schema.toArray(new String[0]);
-    return ColumnPath.get(path);
-  }
-
   // Visible for testing
   MessageType fromParquetSchema(List<SchemaElement> schema, List<ColumnOrder> columnOrders) {
     Iterator<SchemaElement> iterator = schema.iterator();

parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java

@@ -525,7 +525,6 @@ private FileMetaData metadata(long... sizes) {
       columnChunk.setMeta_data(new ColumnMetaData(
           INT32,
           Collections.<org.apache.parquet.format.Encoding>emptyList(),
-          Collections.<String>emptyList(),
           UNCOMPRESSED,
           10l,
           size * 2,

pom.xml

@@ -84,7 +84,7 @@
     <shade.prefix>shaded.parquet</shade.prefix>
     <!-- Guarantees no newer classes/methods/constants are used by parquet. -->
     <hadoop.version>3.3.0</hadoop.version>
-    <parquet.format.version>2.12.0</parquet.format.version>
+    <parquet.format.version>2.13.0-SNAPSHOT-nopath</parquet.format.version>
     <previous.version>1.17.0</previous.version>
     <thrift.executable>thrift</thrift.executable>
     <format.thrift.executable>${thrift.executable}</format.thrift.executable>