Skip to content

Commit 65bc518

Browse files
amousavigourabiamousavigourabi
authored
PARQUET-2359: Add simple plain Parquet configuration implementation (#1182)
1 parent 2b60ffa commit 65bc518

3 files changed

Lines changed: 231 additions & 17 deletions

File tree

‎parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java‎

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.apache.hadoop.fs.Path;
5555
import org.apache.parquet.conf.HadoopParquetConfiguration;
5656
import org.apache.parquet.conf.ParquetConfiguration;
57+
import org.apache.parquet.conf.PlainParquetConfiguration;
5758
import org.apache.parquet.hadoop.ParquetReader;
5859
import org.apache.parquet.hadoop.ParquetWriter;
5960
import org.apache.parquet.hadoop.api.WriteSupport;
@@ -79,31 +80,39 @@ public class TestReadWrite {
7980
@Parameterized.Parameters
8081
public static Collection<Object[]> data() {
8182
Object[][] data = new Object[][] {
82-
{ false, false, false }, // use the new converters with hadoop config
83-
{ true, false, false }, // use the old converters with hadoop config
84-
{ false, true, false }, // use a local disk location with hadoop config
85-
{ false, false, true }, // use the new converters with parquet config interface
86-
{ true, false, true }, // use the old converters with parquet config interface
87-
{ false, true, true } }; // use a local disk location with parquet config interface
83+
{ true, false, false, false }, // use the old converters with hadoop config
84+
{ true, false, true, false }, // use the old converters with parquet config interface
85+
{ false, false, false, false }, // use the new converters with hadoop config
86+
{ false, true, false, false }, // use a local disk location with hadoop config
87+
{ false, false, true, false }, // use the new converters with parquet config interface
88+
{ false, true, true, false }, // use a local disk location with parquet config interface
89+
{ false, false, true, true }, // use the new converters with plain parquet config
90+
{ false, true, true, true } }; // use a local disk location with plain parquet config
8891
return Arrays.asList(data);
8992
}
9093

9194
private final boolean compat;
9295
private final boolean local;
9396
private final boolean confInterface;
97+
private final boolean plainConf;
9498
private final Configuration testConf = new Configuration();
95-
private final ParquetConfiguration parquetConf = new HadoopParquetConfiguration(true);
99+
private final ParquetConfiguration hadoopConfWithInterface = new HadoopParquetConfiguration();
100+
private final ParquetConfiguration plainParquetConf = new PlainParquetConfiguration();
96101

97-
public TestReadWrite(boolean compat, boolean local, boolean confInterface) {
102+
public TestReadWrite(boolean compat, boolean local, boolean confInterface, boolean plainConf) {
98103
this.compat = compat;
99104
this.local = local;
100105
this.confInterface = confInterface;
106+
this.plainConf = plainConf;
101107
this.testConf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, compat);
102108
this.testConf.setBoolean("parquet.avro.add-list-element-records", false);
103109
this.testConf.setBoolean("parquet.avro.write-old-list-structure", false);
104-
this.parquetConf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, compat);
105-
this.parquetConf.setBoolean("parquet.avro.add-list-element-records", false);
106-
this.parquetConf.setBoolean("parquet.avro.write-old-list-structure", false);
110+
this.hadoopConfWithInterface.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, compat);
111+
this.hadoopConfWithInterface.setBoolean("parquet.avro.add-list-element-records", false);
112+
this.hadoopConfWithInterface.setBoolean("parquet.avro.write-old-list-structure", false);
113+
this.plainParquetConf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, compat);
114+
this.plainParquetConf.setBoolean("parquet.avro.add-list-element-records", false);
115+
this.plainParquetConf.setBoolean("parquet.avro.write-old-list-structure", false);
107116
}
108117

109118
@Test
@@ -891,9 +900,15 @@ private ParquetWriter<GenericRecord> writer(String file, Schema schema) throws I
891900
.withSchema(schema);
892901
}
893902
if (confInterface) {
894-
return writerBuilder
895-
.withConf(parquetConf)
896-
.build();
903+
if (plainConf) {
904+
return writerBuilder
905+
.withConf(hadoopConfWithInterface)
906+
.build();
907+
} else {
908+
return writerBuilder
909+
.withConf(plainParquetConf)
910+
.build();
911+
}
897912
} else {
898913
return writerBuilder
899914
.withConf(testConf)
@@ -911,9 +926,15 @@ private ParquetReader<GenericRecord> reader(String file) throws IOException {
911926
return new AvroParquetReader<>(testConf, new Path(file));
912927
}
913928
if (confInterface) {
914-
return readerBuilder
915-
.withConf(parquetConf)
916-
.build();
929+
if (plainConf) {
930+
return readerBuilder
931+
.withConf(hadoopConfWithInterface)
932+
.build();
933+
} else {
934+
return readerBuilder
935+
.withConf(plainParquetConf)
936+
.build();
937+
}
917938
} else {
918939
return readerBuilder
919940
.withConf(testConf)

‎parquet-common/src/main/java/org/apache/parquet/conf/PlainParquetConfiguration.java‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.parquet.conf;
21+
22+
import java.util.HashMap;
23+
import java.util.Iterator;
24+
import java.util.Map;
25+
26+
/**
27+
* Configuration for Parquet without Hadoop dependency.
28+
*/
29+
public class PlainParquetConfiguration implements ParquetConfiguration {
30+
31+
private final Map<String, String> map;
32+
33+
public PlainParquetConfiguration() {
34+
map = new HashMap<>();
35+
}
36+
37+
public PlainParquetConfiguration(Map<String, String> properties) {
38+
map = new HashMap<>(properties);
39+
}
40+
41+
@Override
42+
public void set(String s, String s1) {
43+
map.put(s, s1);
44+
}
45+
46+
@Override
47+
public void setLong(String name, long value) {
48+
set(name, String.valueOf(value));
49+
}
50+
51+
@Override
52+
public void setInt(String name, int value) {
53+
set(name, String.valueOf(value));
54+
}
55+
56+
@Override
57+
public void setBoolean(String name, boolean value) {
58+
set(name, String.valueOf(value));
59+
}
60+
61+
@Override
62+
public void setStrings(String name, String... value) {
63+
if (value.length > 0) {
64+
StringBuilder sb = new StringBuilder(value[0]);
65+
for (int i = 1; i < value.length; ++i) {
66+
sb.append(',');
67+
sb.append(value[i]);
68+
}
69+
set(name, sb.toString());
70+
} else {
71+
set(name, "");
72+
}
73+
}
74+
75+
@Override
76+
public void setClass(String name, Class<?> value, Class<?> xface) {
77+
if (xface.isAssignableFrom(value)) {
78+
set(name, value.getName());
79+
} else {
80+
throw new RuntimeException(xface.getCanonicalName() + " is not assignable from " + value.getCanonicalName());
81+
}
82+
}
83+
84+
@Override
85+
public String get(String name) {
86+
return map.get(name);
87+
}
88+
89+
@Override
90+
public String get(String name, String defaultValue) {
91+
String value = get(name);
92+
if (value != null) {
93+
return value;
94+
} else {
95+
return defaultValue;
96+
}
97+
}
98+
99+
@Override
100+
public long getLong(String name, long defaultValue) {
101+
String value = get(name);
102+
if (value != null) {
103+
return Long.parseLong(value);
104+
} else {
105+
return defaultValue;
106+
}
107+
}
108+
109+
@Override
110+
public int getInt(String name, int defaultValue) {
111+
String value = get(name);
112+
if (value != null) {
113+
return Integer.parseInt(value);
114+
} else {
115+
return defaultValue;
116+
}
117+
}
118+
119+
@Override
120+
public boolean getBoolean(String name, boolean defaultValue) {
121+
String value = get(name);
122+
if (value != null) {
123+
return Boolean.parseBoolean(value);
124+
} else {
125+
return defaultValue;
126+
}
127+
}
128+
129+
@Override
130+
public String getTrimmed(String name) {
131+
String value = get(name);
132+
if (value != null) {
133+
return value.trim();
134+
} else {
135+
return null;
136+
}
137+
}
138+
139+
@Override
140+
public String getTrimmed(String name, String defaultValue) {
141+
String value = get(name);
142+
if (value != null) {
143+
return value.trim();
144+
} else {
145+
return defaultValue;
146+
}
147+
}
148+
149+
@Override
150+
public String[] getStrings(String name, String[] defaultValue) {
151+
String value = get(name);
152+
if (value != null) {
153+
return value.split(",");
154+
} else {
155+
return defaultValue;
156+
}
157+
}
158+
159+
@Override
160+
public Class<?> getClass(String name, Class<?> defaultValue) {
161+
String value = get(name);
162+
if (value != null) {
163+
try {
164+
return Class.forName(value);
165+
} catch (ClassNotFoundException e) {
166+
throw new RuntimeException(e);
167+
}
168+
} else {
169+
return defaultValue;
170+
}
171+
}
172+
173+
@Override
174+
@SuppressWarnings("unchecked")
175+
public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) {
176+
Class<?> value = getClass(name, defaultValue);
177+
if (value != null && value.isAssignableFrom(xface)) {
178+
return (Class<? extends U>) value;
179+
}
180+
return defaultValue;
181+
}
182+
183+
@Override
184+
public Class<?> getClassByName(String name) throws ClassNotFoundException {
185+
return Class.forName(name);
186+
}
187+
188+
@Override
189+
public Iterator<Map.Entry<String, String>> iterator() {
190+
return map.entrySet().iterator();
191+
}
192+
}

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,7 @@
546546
<exclude>org.apache.parquet.hadoop.ColumnChunkPageWriteStore</exclude>
547547
<exclude>org.apache.parquet.hadoop.ParquetRecordWriter</exclude>
548548
<!-- likely japicmp bug, triggers on new interface methods after updating to 0.18.1 -->
549+
<exclude>org.apache.parquet.conf.PlainParquetConfiguration#getClass(java.lang.String,java.lang.Class,java.lang.Class)</exclude>
549550
<exclude>org.apache.parquet.conf.ParquetConfiguration#getClass(java.lang.String,java.lang.Class,java.lang.Class)</exclude>
550551
<exclude>org.apache.parquet.hadoop.util.SerializationUtil#readObjectFromConfAsBase64(java.lang.String,org.apache.parquet.conf.ParquetConfiguration)</exclude>
551552
<exclude>org.apache.parquet.conf.HadoopParquetConfiguration#getClass(java.lang.String,java.lang.Class,java.lang.Class)</exclude>

0 commit comments

Comments
 (0)