Skip to content

Commit ed308ff

Browse files
amousavigourabiamousavigourabi
authored
PARQUET-2419: Reduce noisy logging when running test suite (#1253)
1 parent 8264d8b commit ed308ff

14 files changed

Lines changed: 58 additions & 28 deletions

File tree

‎.github/workflows/ci-hadoop2.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,4 @@ jobs:
5353
run: |
5454
EXTRA_JAVA_TEST_ARGS=$(mvn help:evaluate -Dexpression=extraJavaTestArgs -q -DforceStdout)
5555
export MAVEN_OPTS="$MAVEN_OPTS $EXTRA_JAVA_TEST_ARGS"
56-
mvn verify --batch-mode -P hadoop2 javadoc:javadoc -Pci-test
56+
mvn verify --batch-mode -P hadoop2 javadoc:javadoc

‎.github/workflows/ci-hadoop3.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,4 @@ jobs:
5454
run: |
5555
EXTRA_JAVA_TEST_ARGS=$(mvn help:evaluate -Dexpression=extraJavaTestArgs -q -DforceStdout)
5656
export MAVEN_OPTS="$MAVEN_OPTS $EXTRA_JAVA_TEST_ARGS"
57-
mvn verify --batch-mode javadoc:javadoc -Pci-test
57+
mvn verify --batch-mode javadoc:javadoc

‎.github/workflows/vector-plugins.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,4 @@ jobs:
5454
run: |
5555
EXTRA_JAVA_TEST_ARGS=$(mvn help:evaluate -Dexpression=extraJavaTestArgs -q -DforceStdout)
5656
export MAVEN_OPTS="$MAVEN_OPTS $EXTRA_JAVA_TEST_ARGS"
57-
mvn verify --batch-mode -Pvector-plugins javadoc:javadoc -Pci-test -pl parquet-plugins/parquet-encoding-vector,parquet-plugins/parquet-plugins-benchmarks -am
57+
mvn verify --batch-mode -Pvector-plugins javadoc:javadoc -pl parquet-plugins/parquet-encoding-vector,parquet-plugins/parquet-plugins-benchmarks -am

‎parquet-column/src/test/java/org/apache/parquet/column/mem/TestMemPageStore.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,13 @@
3131
import org.apache.parquet.column.statistics.LongStatistics;
3232
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
3333
import org.junit.Test;
34+
import org.slf4j.Logger;
35+
import org.slf4j.LoggerFactory;
3436

3537
public class TestMemPageStore {
3638

39+
private static final Logger LOG = LoggerFactory.getLogger(TestMemPageStore.class);
40+
3741
private String[] path = {"foo", "bar"};
3842

3943
@Test
@@ -48,12 +52,12 @@ public void test() throws IOException {
4852
pageWriter.writePage(BytesInput.from(new byte[735]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
4953
PageReader pageReader = memPageStore.getPageReader(col);
5054
long totalValueCount = pageReader.getTotalValueCount();
51-
System.out.println(totalValueCount);
55+
LOG.info(String.valueOf(totalValueCount));
5256
int total = 0;
5357
do {
5458
DataPage readPage = pageReader.readPage();
5559
total += readPage.getValueCount();
56-
System.out.println(readPage);
60+
LOG.info(readPage.toString());
5761
// TODO: assert
5862
} while (total < totalValueCount);
5963
}

‎parquet-column/src/test/java/org/apache/parquet/io/ExpectationValidatingRecordConsumer.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ public ExpectationValidatingRecordConsumer(Deque<String> expectations) {
3333
}
3434

3535
private void validate(String got) {
36-
// System.out.println(" \"" + got + "\";");
3736
assertEquals("event #" + count, expectations.pop(), got);
3837
++count;
3938
}

‎parquet-hadoop/src/test/java/org/apache/parquet/encodings/FileEncodingsIT.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@
6464
import org.junit.rules.TemporaryFolder;
6565
import org.junit.runner.RunWith;
6666
import org.junit.runners.Parameterized;
67+
import org.slf4j.Logger;
68+
import org.slf4j.LoggerFactory;
6769

6870
/**
6971
* This class contains test cases to validate each data type encoding.
@@ -72,6 +74,9 @@
7274
*/
7375
@RunWith(Parameterized.class)
7476
public class FileEncodingsIT {
77+
78+
private static final Logger LOG = LoggerFactory.getLogger(FileEncodingsIT.class);
79+
7580
private static final int RANDOM_SEED = 1;
7681
private static final int RECORD_COUNT = 2000000;
7782
private static final int FIXED_LENGTH = 60;
@@ -156,7 +161,7 @@ public void testFileEncodingsWithoutDictionary() throws Exception {
156161
* This loop will make sure to test future writer versions added to WriterVersion enum.
157162
*/
158163
for (WriterVersion writerVersion : WriterVersion.values()) {
159-
System.out.println(String.format(
164+
LOG.info(String.format(
160165
"Testing %s/%s/%s encodings using ROW_GROUP_SIZE=%d PAGE_SIZE=%d",
161166
writerVersion, this.paramTypeName, this.compression, TEST_ROW_GROUP_SIZE, TEST_PAGE_SIZE));
162167

@@ -182,7 +187,7 @@ public void testFileEncodingsWithDictionary() throws Exception {
182187
* This loop will make sure to test future writer versions added to WriterVersion enum.
183188
*/
184189
for (WriterVersion writerVersion : WriterVersion.values()) {
185-
System.out.println(String.format(
190+
LOG.info(String.format(
186191
"Testing %s/%s/%s + DICTIONARY encodings using ROW_GROUP_SIZE=%d PAGE_SIZE=%d",
187192
writerVersion, this.paramTypeName, this.compression, TEST_ROW_GROUP_SIZE, TEST_PAGE_SIZE));
188193

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/DeprecatedInputFormatTest.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,17 @@
6666
import org.apache.parquet.schema.MessageTypeParser;
6767
import org.junit.Before;
6868
import org.junit.Test;
69+
import org.slf4j.Logger;
70+
import org.slf4j.LoggerFactory;
6971

7072
/**
7173
* DeprecatedParquetInputFormat is used by cascading. It initializes the recordReader using an initialize method with
7274
* different parameters than ParquetInputFormat
7375
*/
7476
public class DeprecatedInputFormatTest {
77+
78+
private static final Logger LOG = LoggerFactory.getLogger(DeprecatedInputFormatTest.class);
79+
7580
final Path parquetPath = new Path("target/test/example/TestInputOutputFormat/parquet");
7681
final Path inputPath = new Path("src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java");
7782
final Path outputPath = new Path("target/test/example/TestInputOutputFormat/out");
@@ -317,10 +322,10 @@ public void testReadWriteWithoutCounter() throws Exception {
317322

318323
private void waitForJob(Job job) throws InterruptedException, IOException {
319324
while (!job.isComplete()) {
320-
System.out.println("waiting for job " + job.getJobName());
325+
LOG.info("waiting for job " + job.getJobName());
321326
sleep(100);
322327
}
323-
System.out.println("status for job " + job.getJobName() + ": " + (job.isSuccessful() ? "SUCCESS" : "FAILURE"));
328+
LOG.info("status for job " + job.getJobName() + ": " + (job.isSuccessful() ? "SUCCESS" : "FAILURE"));
324329
if (!job.isSuccessful()) {
325330
throw new RuntimeException("job failed " + job.getJobName());
326331
}

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestDirectCodecFactory.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,13 @@
3636
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
3737
import org.junit.Assert;
3838
import org.junit.Test;
39+
import org.slf4j.Logger;
40+
import org.slf4j.LoggerFactory;
3941

4042
public class TestDirectCodecFactory {
4143

44+
private static final Logger LOG = LoggerFactory.getLogger(TestDirectCodecFactory.class);
45+
4246
private enum Decompression {
4347
ON_HEAP,
4448
OFF_HEAP,
@@ -121,7 +125,7 @@ private void test(int size, CompressionCodecName codec, boolean useOnHeapCompres
121125
final String msg = String.format(
122126
"Failure while testing Codec: %s, OnHeapCompressionInput: %s, Decompression Mode: %s, Data Size: %d",
123127
codec.name(), useOnHeapCompression, decomp.name(), size);
124-
System.out.println(msg);
128+
LOG.error(msg);
125129
throw new RuntimeException(msg, e);
126130
} finally {
127131
if (rawBuf != null) {

‎parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,13 @@
7171
import org.junit.Rule;
7272
import org.junit.Test;
7373
import org.junit.rules.TemporaryFolder;
74+
import org.slf4j.Logger;
75+
import org.slf4j.LoggerFactory;
7476

7577
public class TestStatistics {
78+
79+
private static final Logger LOG = LoggerFactory.getLogger(TestStatistics.class);
80+
7681
private static final int MEGABYTE = 1 << 20;
7782
private static final long RANDOM_SEED = 1441990701846L; // System.currentTimeMillis();
7883

@@ -481,7 +486,7 @@ public void testStatistics() throws IOException {
481486
File file = folder.newFile("test_file.parquet");
482487
file.delete();
483488

484-
System.out.println(String.format("RANDOM SEED: %s", RANDOM_SEED));
489+
LOG.info(String.format("RANDOM SEED: %s", RANDOM_SEED));
485490

486491
Random random = new Random(RANDOM_SEED);
487492

‎parquet-pig/src/test/java/org/apache/parquet/pig/TestParquetLoader.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,13 @@
4646
import org.apache.pig.impl.logicalLayer.FrontendException;
4747
import org.junit.Assert;
4848
import org.junit.Test;
49+
import org.slf4j.Logger;
50+
import org.slf4j.LoggerFactory;
4951

5052
public class TestParquetLoader {
53+
54+
private static final Logger LOG = LoggerFactory.getLogger(TestParquetLoader.class);
55+
5156
@Test
5257
public void testSchema() throws Exception {
5358
String location = "garbage";
@@ -243,7 +248,7 @@ public void testTypePersuasion() throws Exception {
243248
+ DataType.findTypeName(types[(i + 4) % types.length]) + "," + " b:"
244249
+ DataType.findTypeName(types[(i + 5) % types.length]) + "');";
245250

246-
System.out.println("Query: " + query);
251+
LOG.info("Query: " + query);
247252
pigServer.registerQuery(query);
248253
pigServer.registerQuery("STORE B into 'out" + i + "' using mock.Storage();");
249254
pigServer.executeBatch();

0 commit comments

Comments
 (0)