diff --git a/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetReaderUtils.java b/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetReaderUtils.java index f0a0c9b3d..2d21a2fa1 100644 --- a/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetReaderUtils.java +++ b/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetReaderUtils.java @@ -20,7 +20,6 @@ package io.milvus.bulkwriter.common.utils; import org.apache.avro.generic.GenericData; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.avro.AvroParquetReader; @@ -31,7 +30,7 @@ public void readParquet(String localFilePath) throws IOException { Path path = new Path(localFilePath); try (org.apache.parquet.hadoop.ParquetReader reader = AvroParquetReader .builder(path) - .withConf(new Configuration()) + .withConf(ParquetUtils.getParquetConfiguration()) .build()) { GenericData.Record record; while ((record = reader.read()) != null) { diff --git a/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetUtils.java b/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetUtils.java index 8489cf479..f40a253c6 100644 --- a/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetUtils.java +++ b/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/common/utils/ParquetUtils.java @@ -19,6 +19,7 @@ package io.milvus.bulkwriter.common.utils; +import org.apache.hadoop.conf.Configuration; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType; @@ -158,4 +159,11 @@ private static void fillArrayType(Types.MessageTypeBuilder messageTypeBuilder, C break; } } + + public static Configuration getParquetConfiguration() { + // set fs.file.impl.disable.cache to true for this issue: https://github.com/milvus-io/milvus-sdk-java/issues/1381 + Configuration configuration = new Configuration(); + configuration.set("fs.file.impl.disable.cache", "true"); + return configuration; + } } diff --git a/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/writer/ParquetFileWriter.java b/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/writer/ParquetFileWriter.java index eb622a74b..cd260f5ef 100644 --- a/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/writer/ParquetFileWriter.java +++ b/sdk-bulkwriter/src/main/java/io/milvus/bulkwriter/writer/ParquetFileWriter.java @@ -53,7 +53,7 @@ private void initWriter() throws IOException { int rowGroupBytes = 16 * 1024 * 1024; // declare and define the ParquetWriter. - Configuration configuration = new Configuration(); + Configuration configuration = ParquetUtils.getParquetConfiguration(); GroupWriteSupport.setSchema(messageType, configuration); GroupWriteSupport writeSupport = new GroupWriteSupport(); this.writer = new ParquetWriter<>(new Path(filePath),