S3 Out - Add ZSTD compression when writing Parquet files

Author: gouyelliot

src/aws/compression/arrow/compress.c

@@ -174,6 +174,7 @@ static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table)
 {
         GArrowResizableBuffer *buffer;
         GArrowBufferOutputStream *sink;
+        GParquetWriterProperties *builder;
         GParquetArrowFileWriter *writer;
         GArrowSchema *schema;
         GError *error = NULL;
@@ -199,12 +200,22 @@ static GArrowResizableBuffer* table_to_parquet_buffer(GArrowTable *table)
             return NULL;
         }
 
+        builder = gparquet_writer_properties_new();
+        if (builder == NULL) {
+            g_object_unref(buffer);
+            g_object_unref(sink);
+            g_object_unref(schema);
+            return NULL;
+        }
+        gparquet_writer_properties_set_compression(builder, GARROW_COMPRESSION_TYPE_ZSTD, NULL);
+
         /* Create a new Parquet file writer */
         writer = gparquet_arrow_file_writer_new_arrow(schema,
                                                       GARROW_OUTPUT_STREAM(sink),
-                                                      NULL, /* Arrow writer properties */
+                                                      builder, /* Arrow writer properties */
                                                       &error);
         g_object_unref(schema);
+        g_object_unref(builder);
         if (writer == NULL) {
             flb_error("[aws][compress] Failed to create parquet writer: %s", error->message);
             g_error_free(error);