[553] Parquet Schema and Column Stats Converters (#669)

* smaller PR for parquet

* read parquet file for metadataExtractor: compiling, not testd

* cleanups for statsExtractor: compiling, not testd

* refactoring for statsExtractor: compiling, not testd

* added avro dependency

* added tests for SchemaExtractor: int and string primitiveTypes test passes

* fixed some minor bugs in SchemaExtractor

* close fileReader and handle exception

* adjusted fromInternalSchema()

* added a test and adjusted SchemaExtractor

* added a testing code

* bug fix for Schema extractor: groupType

* bug fix for Schema extractor

* bug fix for tests

* bug fix for SchemaExtractor and added tests for nested lists support

* bug fix for tests for nested lists support

* bug fix for complex test which now passes!

* added test for Map

* schemaExtractor refactored

* bug fixed isNullable() schema

* fromInternalSchema : list and map types

* decimal primitive test added

* float primitive + list and map tests for fromInternalSchema

* added tests for primitive type (date and timestamp)

* refactoring for partitionValues extractor

* git build error fixed

* cleanups for schemaExtractor + refactoring for schemaExtractorTests + added test code for statsExtractor

* added assertsEqual test for stats + removed partitionFields from the test,  TODO check if field is needed in ColumnStats

* bug fixed for stats tests: columnStats + tests data are read using FileReader

* bug fixed for stats tests, TODO equality test for two objects

* added compareFiles() in InternDataFile for the statsExtractor tests to pass: OK

* added custom comparison test for ColumnStat and InternDataFile, test passes, TODO: other stat types and other schema types testing

* added custom comparison test for ColumnStat (field) and exec spotless apply

* tempDir for parquet stats testing

* binaryStatistics test passes

* added int32 file schema test for statsExtractor

* cleanups + added fields comparison for InternalDataFile

* cleanups + added fixed_len_byte_array primitive type schema file test

* use of genericGetMax instead for stats extraction + cleanups

* boolean schema file test for statsExtractor added

* removed hard coded path in statsExtractor test

* cleanups + imports

* separate tests for int and binary for stats

* custom equals() not needed for InternalDataFile and ColumnStat

* removed parquet version from core sub-project pom

* statsExtractor tests as a suite, removed comments + run spotless apply

* removed uncessary classes

* removed uncessary classes: undo

* undo irrelevant changes

* fixed formatting issues with spotless:apply cmd

* cleanups for test class and fixes for failed build

* tmp file name fixed for failed build

* cleanups

* splotless apply run + assertion internalDataFile equality changed to display errors

* fixes for build, PhysicalPath and BinaryStats

* fixes for build, PhysicalPath and BinaryStats + synced fork

* fixes for build, PhysicalPath and BinaryStats + synced fork

* fixes for build and cleanups

* fixes for build and cleanups

* Parquet dep set as provided to use Spark's

* parquet dep version back to 1.15.1

* parquet-avro moved from core to project's pom

* parquet-avro moved after hadoop-common

* parquet dep scope removed

* run spotless:apply

---------

Co-authored-by: Selim Soufargi <ssoufargi.idealab.unical@gmail.com~>

Author: sapienza88

xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java

@@ -34,12 +34,16 @@
 class ExternalTable {
   /** The name of the table. */
   protected final @NonNull String name;
+
   /** The format of the table (e.g. DELTA, ICEBERG, HUDI) */
   protected final @NonNull String formatName;
+
   /** The path to the root of the table or the metadata directory depending on the format */
   protected final @NonNull String basePath;
+
   /** Optional namespace for the table */
   protected final String[] namespace;
+
   /** The configuration for interacting with the catalog that manages this table */
   protected final CatalogConfig catalogConfig;
 

xtable-api/src/main/java/org/apache/xtable/model/catalog/ThreePartHierarchicalTableIdentifier.java

@@ -42,6 +42,7 @@ public class ThreePartHierarchicalTableIdentifier implements HierarchicalTableId
    * name varies depending on the catalogType.
    */
   String catalogName;
+
   /**
    * Catalogs have the ability to group tables logically, databaseName is the identifier for such
    * logical classification. The alternate names for this field include namespace, schemaName etc.

xtable-api/src/main/java/org/apache/xtable/model/schema/InternalField.java

@@ -43,9 +43,11 @@ public class InternalField {
   // The id field for the field. This is used to identify the field in the schema even after
   // renames.
   Integer fieldId;
+
   // represents the fully qualified path to the field (dot separated)
   @Getter(lazy = true)
   String path = createPath();
+
   // splits the dot separated path into parts
   @Getter(lazy = true)
   String[] pathParts = splitPath();

xtable-api/src/main/java/org/apache/xtable/model/schema/InternalSchema.java

@@ -75,7 +75,8 @@ public enum MetadataKey {
 
   public enum MetadataValue {
     MICROS,
-    MILLIS
+    MILLIS,
+    NANOS
   }
 
   public static final String XTABLE_LOGICAL_TYPE = "xtableLogicalType";

xtable-core/pom.xml

@@ -56,7 +56,7 @@
             <artifactId>guava</artifactId>
         </dependency>
 
-        <!-- Avro -->
+
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
@@ -116,6 +116,10 @@
             <artifactId>hadoop-common</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-avro</artifactId>
+        </dependency>
 
         <!-- Logging API -->
         <dependency>

xtable-core/src/main/java/org/apache/xtable/parquet/ParquetMetadataExtractor.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.parquet;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.HadoopReadOptions;
+import org.apache.parquet.ParquetReadOptions;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.apache.parquet.io.InputFile;
+import org.apache.parquet.schema.MessageType;
+
+import org.apache.xtable.exception.ReadException;
+
+public class ParquetMetadataExtractor {
+
+  private static final ParquetMetadataExtractor INSTANCE = new ParquetMetadataExtractor();
+
+  public static ParquetMetadataExtractor getInstance() {
+    return INSTANCE;
+  }
+
+  public static MessageType getSchema(ParquetMetadata footer) {
+    MessageType schema = footer.getFileMetaData().getSchema();
+    return schema;
+  }
+
+  public static ParquetMetadata readParquetMetadata(Configuration conf, Path filePath) {
+    InputFile file = null;
+    try {
+      file = HadoopInputFile.fromPath(filePath, conf);
+    } catch (IOException e) {
+      throw new ReadException("Failed to read the parquet file", e);
+    }
+
+    ParquetReadOptions options = HadoopReadOptions.builder(conf, filePath).build();
+    try (ParquetFileReader fileReader = ParquetFileReader.open(file, options)) {
+      return fileReader.getFooter();
+    } catch (Exception e) {
+      throw new ReadException("Failed to read the parquet file", e);
+    }
+  }
+}