update

Author: ArnavBalyan

parquet-cli/README.md

@@ -134,3 +134,25 @@ Usage: parquet [options] [command] [command options]
   See 'parquet help <command>' for more information on a specific command.
 ```
 
+### Configuration Options
+
+The `convert` and `convert-csv` commands support a generic configuration mechanism:
+
+- `--conf` or `--property`: Set any configuration property in format `key=value`. Can be specified multiple times.
+
+This allows you to configure any Avro or Parquet setting without needing to modify source code or rebuild the project.
+
+Examples:
+```bash
+# Enable UUID support
+parquet convert input.avro -o output.parquet --conf parquet.avro.write-parquet-uuid=true
+
+# Use new 3-level list structure
+parquet convert input.avro -o output.parquet --conf parquet.avro.write-old-list-structure=false
+
+# Convert CSV with multiple options
+parquet convert-csv input.csv -o output.parquet --schema schema.avsc --conf parquet.avro.write-parquet-uuid=true --conf parquet.avro.write-old-list-structure=false
+
+# Set any other configuration properties
+parquet convert input.avro -o output.parquet --conf parquet.avro.add-list-element-records=false --conf parquet.avro.write.data.supplier=org.apache.parquet.avro.GenericDataSupplier
+```

parquet-cli/src/main/java/org/apache/parquet/cli/commands/ConvertCSVCommand.java

@@ -45,6 +45,7 @@
 import org.apache.parquet.cli.util.Schemas;
 import org.apache.parquet.hadoop.ParquetFileWriter;
 import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.hadoop.conf.Configuration;
 import org.slf4j.Logger;
 
 @Parameters(commandDescription = "Create a file from CSV data")
@@ -117,6 +118,11 @@ public ConvertCSVCommand(Logger console) {
       description = "Remove any data already in the target view or dataset")
   boolean overwrite = false;
 
+  @Parameter(
+      names = {"--conf", "--property"},
+      description = "Set a configuration property (format: key=value). Can be specified multiple times.")
+  List<String> confProperties;
+
   @Override
   @SuppressWarnings("unchecked")
   public int run() throws IOException {
@@ -168,6 +174,23 @@ public int run() throws IOException {
       }
     }
 
+    // Create a configuration and apply custom properties
+    Configuration conf = new Configuration(getConf());
+
+    // Apply custom configuration properties
+    if (confProperties != null) {
+      for (String prop : confProperties) {
+        String[] parts = prop.split("=", 2);
+        if (parts.length != 2) {
+          throw new IllegalArgumentException("Configuration property must be in format key=value: " + prop);
+        }
+        String key = parts[0].trim();
+        String value = parts[1].trim();
+        conf.set(key, value);
+        console.debug("Set configuration property: {}={}", key, value);
+      }
+    }
+
     try (ParquetWriter<Record> writer = AvroParquetWriter.<Record>builder(qualifiedPath(outputPath))
         .withWriterVersion(v2 ? PARQUET_2_0 : PARQUET_1_0)
         .withWriteMode(overwrite ? ParquetFileWriter.Mode.OVERWRITE : ParquetFileWriter.Mode.CREATE)
@@ -177,7 +200,7 @@ public int run() throws IOException {
         .withPageSize(pageSize)
         .withRowGroupSize(rowGroupSize)
         .withDataModel(GenericData.get())
-        .withConf(getConf())
+        .withConf(conf)
         .withSchema(csvSchema)
         .build()) {
       for (String target : targets) {

parquet-cli/src/main/java/org/apache/parquet/cli/commands/ConvertCommand.java

@@ -43,6 +43,7 @@
 import org.apache.parquet.hadoop.ParquetWriter;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 import org.slf4j.Logger;
+import org.apache.hadoop.conf.Configuration;
 
 @Parameters(commandDescription = "Create a Parquet file from a data file")
 public class ConvertCommand extends BaseCommand {
@@ -95,6 +96,11 @@ public ConvertCommand(Logger console) {
   @Parameter(names = "--dictionary-size", description = "Max dictionary page size")
   int dictionaryPageSize = ParquetWriter.DEFAULT_PAGE_SIZE;
 
+  @Parameter(
+      names = {"--conf", "--property"},
+      description = "Set a configuration property (format: key=value). Can be specified multiple times.")
+  List<String> confProperties;
+
   @Override
   @SuppressWarnings("unchecked")
   public int run() throws IOException {
@@ -119,13 +125,30 @@ public int run() throws IOException {
       outFS.delete(outPath);
     }
 
+    // Create a configuration and apply custom properties
+    Configuration conf = new Configuration(getConf());
+
+    // Apply custom configuration properties
+    if (confProperties != null) {
+      for (String prop : confProperties) {
+        String[] parts = prop.split("=", 2);
+        if (parts.length != 2) {
+          throw new IllegalArgumentException("Configuration property must be in format key=value: " + prop);
+        }
+        String key = parts[0].trim();
+        String value = parts[1].trim();
+        conf.set(key, value);
+        console.debug("Set configuration property: {}={}", key, value);
+      }
+    }
+
     Iterable<Record> reader = openDataFile(source, projection);
     boolean threw = true;
     long count = 0;
     try {
       try (ParquetWriter<Record> writer = AvroParquetWriter.<Record>builder(qualifiedPath(outputPath))
           .withWriterVersion(v2 ? PARQUET_2_0 : PARQUET_1_0)
-          .withConf(getConf())
+          .withConf(conf)
           .withCompressionCodec(codec)
           .withRowGroupSize(rowGroupSize)
           .withDictionaryPageSize(dictionaryPageSize < 64 ? 64 : dictionaryPageSize)

parquet-cli/src/test/java/org/apache/parquet/cli/commands/ConvertCSVCommandTest.java

@@ -61,4 +61,20 @@ public void testConvertCSVCommandWithDifferentSchemas() throws IOException {
     command.setConf(new Configuration());
     command.run();
   }
+
+  @Test
+  public void testConvertCSVCommandWithGenericConf() throws IOException {
+    File file = csvFile();
+    ConvertCSVCommand command = new ConvertCSVCommand(createLogger());
+    command.targets = Arrays.asList(file.getAbsolutePath());
+    File output = new File(getTempFolder(), getClass().getSimpleName() + "_with_generic_conf.parquet");
+    command.outputPath = output.getAbsolutePath();
+    command.confProperties = Arrays.asList(
+        "parquet.avro.write-parquet-uuid=true",
+        "parquet.avro.write-old-list-structure=false"
+    );
+    command.setConf(new Configuration());
+    Assert.assertEquals(0, command.run());
+    Assert.assertTrue(output.exists());
+  }
 }

parquet-cli/src/test/java/org/apache/parquet/cli/commands/ConvertCommandTest.java

@@ -37,4 +37,63 @@ public void testConvertCommand() throws IOException {
     Assert.assertEquals(0, command.run());
     Assert.assertTrue(output.exists());
   }
+
+  @Test
+  public void testConvertCommandWithGenericConf() throws IOException {
+    File file = toAvro(parquetFile());
+    ConvertCommand command = new ConvertCommand(createLogger());
+    command.targets = Arrays.asList(file.getAbsolutePath());
+    File output = new File(getTempFolder(), "converted_with_generic_conf.parquet");
+    command.outputPath = output.getAbsolutePath();
+    command.confProperties = Arrays.asList(
+        "parquet.avro.write-parquet-uuid=true",
+        "parquet.avro.write-old-list-structure=false",
+        "test.property=test.value"
+    );
+    command.setConf(new Configuration());
+
+    Assert.assertEquals(0, command.run());
+    Assert.assertTrue(output.exists());
+  }
+
+  @Test
+  public void testConvertCommandConfigurationValidation() throws IOException {
+    File file = toAvro(parquetFile());
+    ConvertCommand command = new ConvertCommand(createLogger());
+    command.targets = Arrays.asList(file.getAbsolutePath());
+    File output = new File(getTempFolder(), "converted_with_config_validation.parquet");
+    command.outputPath = output.getAbsolutePath();
+
+    command.confProperties = Arrays.asList(
+        "parquet.avro.write-parquet-uuid=true",
+        "parquet.avro.write-old-list-structure=false"
+    );
+
+    command.setConf(new Configuration());
+
+    Assert.assertEquals(0, command.run());
+    Assert.assertTrue(output.exists());
+
+    File output2 = new File(getTempFolder(), "converted_with_config_validation2.parquet");
+    command.outputPath = output2.getAbsolutePath();
+    command.confProperties = Arrays.asList(
+        "parquet.avro.write-parquet-uuid=false",
+        "parquet.avro.write-old-list-structure=true"
+    );
+
+    Assert.assertEquals(0, command.run());
+    Assert.assertTrue(output2.exists());
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testConvertCommandWithInvalidConf() throws IOException {
+    File file = toAvro(parquetFile());
+    ConvertCommand command = new ConvertCommand(createLogger());
+    command.targets = Arrays.asList(file.getAbsolutePath());
+    File output = new File(getTempFolder(), "converted_with_invalid_conf.parquet");
+    command.outputPath = output.getAbsolutePath();
+    command.confProperties = Arrays.asList("invalid-property-format");
+    command.setConf(new Configuration());
+    command.run();
+  }
 }